build http cache proxy protocol

1_7.http_proxy_server/python/my-twisted-connect-proxy/CacheUtils.py

@@ -4,7 +4,63 @@ import urllib
 import urllib2
 import json
 class CacheUtils:
-    @staticmethod
+    def download(self, url, local):
+        #urllib.urlretrieve(url, local, self.cbk)
+        urllib.urlretrieve(url, local)
+
+    def parseUrl2FileName(self, url):
+        fileName = url.split('/')[-1]
+        #deal with pure url
+        if fileName == "":
+            fileName = url.split("//")[-1].replace('/', '')
+        return fileName
+
+    def cache(self, url, range):
+        fileName = self.parseUrl2FileName(url)
+        req = urllib2.Request(url)
+        req.add_header('Range', 'bytes=' + range)
+        response = urllib2.urlopen(req)
+        buffer = response.read()
+        with open("./cache/" + fileName + range, "a+") as fp:
+            fp.write(buffer)
+
+    def saveReq(self, url, range):
+        # Reading data back
+        with open('data.json', 'r') as fp:
+            data = json.load(fp)
+        data[url] = range
+        # Writing JSON data
+        with open('data.json', 'w') as fp:
+            json.dump(data, fp)
+    
+    def delReq(sel, url):
+        # Reading data back
+        with open('data.json', 'r') as fp:
+            data = json.load(fp)
+        if data.get(url):
+            del data[url]
+        # Writing JSON data
+        with open('data.json', 'w') as fp:
+            json.dump(data, fp)
+
+    def checkReq(self, url):
+        # Reading data back
+        with open('data.json', 'r') as fp:
+            data = json.load(fp)
+        if data.get(url):
+            fileName = url.split('/')[-1]
+            with open('GotIt.txt', 'a+') as fp:
+                if data[url] == "None":
+                    fp.write("the file you request has been downloaded: ./download/" + fileName + "\n")
+                else:
+                    fp.write("the file you request has been cached: ./cache/" + fileName + "  " + data[url] + "\n")
+            return True
+        else:
+            return False
+
+
+"""
+ @staticmethod
     def cbk(a, b, c):  
         '''''回调函数 
         @a: 已经下载的数据块 
@@ -16,45 +72,9 @@ class CacheUtils:
             per = 100  
         print '%.2f%%' % per  
 
-    def download(self, url, local):
-        urllib.urlretrieve(url, local, self.cbk)
-
-    def cache(self, url, range):
-        fileName = url.split('/')[-1]
-        req = urllib2.Request(url)
-        req.add_header('Range', 'bytes=' + range)
-        response = urllib2.urlopen(req)
-        buffer = response.read()
-        with open("./cache/" + fileName + range, "a+") as fp:
-            fp.write(buffer)
-
-    def saveReq(self, url):
-
-        # Reading data back
-        with open('data.json', 'r') as fp:
-            data = json.load(fp)
-        data[url] = 4000
-        # Writing JSON data
-        with open('data.json', 'w') as fp:
-            json.dump(data, fp)
-   
-    
-    def checkReq(self):
-        # Reading data back
-        with open('data.json', 'r') as fp:
-            data = json.load(fp)
-        #print(data)
-        #print(data.keys())
-        print(data["www.baidu.com"])
-        if data.get("key"): 
-            print(data["key"])
-        else:
-            print("error")
-
-"""
 if __name__ == '__main__':
     cacheUtils = CacheUtils()
-    
+    cacheUtils.delReq("http://static.youku.com/v1.0.1029/cms/img/zy.png")
     #url = "http://www.sina.com.cn"
     #fileName = url.split('/')[-1]
     #cacheUtils.download(url, "./cache/" + fileName)
@@ -63,7 +83,8 @@ if __name__ == '__main__':
     #cacheUtils.cache("https://ss0.bdstatic.com/5aV1bjqh_Q23odCf/static/superplus/img/logo_white_ee663702.png", "0-7000")
     #cacheUtils.cache("https://ss0.bdstatic.com/5aV1bjqh_Q23odCf/static/superplus/img/logo_white_ee663702.png", "7001-14175")
 
-    cacheUtils.saveReq("http://www.sina.com.cn")
+    
+    #cacheUtils.saveReq("http://www.sina.com.cn")
     
     #cacheUtils.loadReq()
 """

1_7.http_proxy_server/python/my-twisted-connect-proxy/GotIt.txt

@@ -0,0 +1,11 @@
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000

1_7.http_proxy_server/python/my-twisted-connect-proxy/GotIt.txt~

@@ -0,0 +1,80 @@
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/  0-7000
+the file you request has been cached: ./cache/50  0-7000
+the file you request has been cached: ./cache/0130391F4854DE23ABDA71085AD8AF4040077C-72D6-D10E-A8E2-281A83037160  0-7000
+the file you request has been cached: ./cache/0130391F48533B991848F6139B1FA448622DEF-D1C2-EF5A-894E-76ACA2B7040B  0-7000
+the file you request has been cached: ./cache/05100000557A941567BC3D7F2B0633B8  0-7000
+the file you request has been cached: ./cache/05100000546F10836737B34B7E03A098  0-7000
+the file you request has been cached: ./cache/05100000556821A267BC3D2CB10AD9A2  0-7000
+the file you request has been cached: ./cache/05100000556821B667BC3D2ABE042C2E  0-7000
+the file you request has been cached: ./cache/051000005465C14F6737B3325A0955CE  0-7000
+the file you request has been cached: ./cache/05100000547705646737B340AA0A959D  0-7000
+the file you request has been cached: ./cache/0130391F4553E22B8C361A0646F416ED804189-16D1-22B0-6592-2DE9528C4EBC  0-7000
+the file you request has been cached: ./cache/0130391F45530DBD99CBF3150EABFA52F54356-2481-BECE-753B-DC8FAD62B933  0-7000
+the file you request has been cached: ./cache/0130391F4554F20EC3A786189FF021190DA8B8-2BDB-69B6-1264-95CA09BEB5AC  0-7000
+the file you request has been cached: ./cache/0130391F485236FCB7B2B50504571BC3F0882E-1B2F-29BF-4FB3-546E15DFD5E5  0-7000
+the file you request has been cached: ./cache/0130391F4550C6E8234FC20830E1366754DB3E-08E4-6423-5297-D1C992589702  0-7000
+the file you request has been cached: ./cache/0130391F455463233C4D1A1BA985B0A18CFBF7-9827-5003-58E8-966A559DED6C  0-7000
+the file you request has been cached: ./cache/0130391F45533B8CE4A5E7030C71A035A2D08D-1CBC-D104-230C-A9F53170EE77  0-7000
+the file you request has been cached: ./cache/0130391F4554C59AB6E5630023C04A1E86AF52-68DD-5B50-DDC7-3543E9FB93EB  0-7000
+the file you request has been cached: ./cache/0130391F4554DC5277E951181CBEAD9761A8B9-B30B-B81C-5B88-F6AC54AB1057  0-7000
+the file you request has been cached: ./cache/0130391F4553BDFD01FB0D04A0DAFB54F6F014-8FDF-60AD-0854-2F756AC1692C  0-7000
+the file you request has been cached: ./cache/yk.css  0-7000
+the file you request has been cached: ./cache/grid_pc.css  0-7000
+the file you request has been cached: ./cache/ykhome.css  0-7000
+the file you request has been cached: ./cache/jquery.js  0-7000
+the file you request has been cached: ./cache/prototype.js  0-7000
+the file you request has been cached: ./cache/common.js  0-7000
+the file you request has been cached: ./cache/cmsCommon.js  0-7000
+the file you request has been cached: ./cache/cmsDatafromPrototype.js  0-7000
+the file you request has been cached: ./cache/compiler.js  0-7000
+the file you request has been cached: ./cache/resize.js  0-7000
+the file you request has been cached: ./cache/qheader.css  0-7000
+the file you request has been cached: ./cache/user-grade-icon.css  0-7000
+the file you request has been cached: ./cache/chuda.js  0-7000
+the file you request has been cached: ./cache/qheader.js  0-7000
+the file you request has been cached: ./cache/qwindow.js  0-7000
+the file you request has been cached: ./cache/popup.js  0-7000
+the file you request has been cached: ./cache/lsidetoolresize.js  0-7000
+the file you request has been cached: ./cache/lsidetool.css  0-7000
+the file you request has been cached: ./cache/lsidetool.js  0-7000
+the file you request has been cached: ./cache/cmsFriends.js  0-7000
+the file you request has been cached: ./cache/seedVideo.js  0-7000
+the file you request has been cached: ./cache/youku_laifeng_v3.js  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/qfooter.css  0-7000
+the file you request has been cached: ./cache/iresearch.js  0-7000
+the file you request has been cached: ./cache/cps.js  0-7000
+the file you request has been cached: ./cache/sideTool.css  0-7000
+the file you request has been cached: ./cache/sideTool.js  0-7000
+the file you request has been cached: ./cache/tdstat.js  0-7000
+the file you request has been cached: ./cache/ani.js  0-7000
+the file you request has been cached: ./cache/hover.js  0-7000
+the file you request has been cached: ./cache/gridTab.js  0-7000
+the file you request has been cached: ./cache/ykRecommend.js  0-7000
+the file you request has been cached: ./cache/chuda.css  0-7000
+the file you request has been cached: ./cache/share_msg.js  0-7000
+the file you request has been cached: ./cache/get.json  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/;stok=a094fa1c55f2125e2a34caadef198da8  0-7000
+the file you request has been cached: ./cache/beacon.js  0-7000
+the file you request has been cached: ./cache/MTFlashStore.swf  0-7000
+the file you request has been cached: ./cache/html  0-7000
+the file you request has been cached: ./cache/loader.swf  0-7000
+the file you request has been cached: ./cache/0130391F45540F219F90571975D51733572DE9-180A-C171-DDB1-75757E59B4F4  0-7000
+the file you request has been cached: ./cache/yklogo_h.png  0-7000
+the file you request has been cached: ./cache/header.png  0-7000
+the file you request has been cached: ./cache/vip.png  0-7000
+the file you request has been cached: ./cache/toolbar.png  0-7000
+the file you request has been cached: ./cache/050C00005583B8C567BC3D6F5E07F183  0-7000
+the file you request has been cached: ./cache/051500005583850C67BC3D22B40DD3C4  0-7000
+the file you request has been cached: ./cache/051500005583904D67BC3D29510BD425  0-7000
+the file you request has been cached: ./cache/list.json  0-7000
+the file you request has been cached: ./cache/getRankData  0-7000