From o2on-svn ¡÷ lists.sourceforge.jp Tue Feb 16 22:04:08 2010 From: o2on-svn ¡÷ lists.sourceforge.jp (o2on svn commit) Date: Tue, 16 Feb 2010 22:04:08 +0900 Subject: [o2on-svn] [178] Check if xml was parsable. Message-ID: <1266325448.093042.5280.nullmailer@users.sourceforge.jp> Revision: 178 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=178 Author: nawota Date: 2010-02-16 22:04:08 +0900 (Tue, 16 Feb 2010) Log Message: ----------- Check if xml was parsable. Modified Paths: -------------- trunk/opy2on/lib/o2on_node.py Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2009-11-07 07:04:58 UTC (rev 177) +++ trunk/opy2on/lib/o2on_node.py 2010-02-16 13:04:08 UTC (rev 178) @@ -250,7 +250,8 @@ pass else: if data: - dom = xml.dom.minidom.parseString(data) + try: dom = xml.dom.minidom.parseString(data) + except: return result nn = dom.getElementsByTagName("nodes") if len(nn): for n in nn[0].getElementsByTagName("node"): From o2on-svn ¡÷ lists.sourceforge.jp Tue Feb 16 22:15:15 2010 From: o2on-svn ¡÷ lists.sourceforge.jp (o2on svn commit) Date: Tue, 16 Feb 2010 22:15:15 +0900 Subject: [o2on-svn] [179] Check if read is completed. Message-ID: <1266326115.928224.23133.nullmailer@users.sourceforge.jp> Revision: 179 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=179 Author: nawota Date: 2010-02-16 22:15:15 +0900 (Tue, 16 Feb 2010) Log Message: ----------- Check if read is completed. Modified Paths: -------------- trunk/opy2on/lib/o2on_node.py Modified: trunk/opy2on/lib/o2on_node.py =================================================================== --- trunk/opy2on/lib/o2on_node.py 2010-02-16 13:04:08 UTC (rev 178) +++ trunk/opy2on/lib/o2on_node.py 2010-02-16 13:15:15 UTC (rev 179) @@ -246,7 +246,7 @@ data = r.read(l) elif r.status == 404: pass else: raise Exception("status error %d" % r.status) - except socket.error: + except socket.error, httplib.IncompleteRead: pass else: if data: @@ -279,8 +279,11 @@ r = self.request("POST","/dat", data,headers) else: r = self.request("GET","/dat",None,headers) - stat = r.status - data = r.read() + try: + stat = r.status + data = r.read() + except socket.error, httplib.IncompleteRead: + return None if stat == 200: path = None path = r.getheader("X-O2-Original-DAT-URL") @@ -303,7 +306,10 @@ l = r.getheader('Content-Length') if l: l = int(l) else: return False - return r.read(l) + try: + return r.read(l) + except socket.error, httplib.IncompleteRead: + return False else: raise Exception("ping status %d" % r.status) return False def collection(self, glob): @@ -313,7 +319,10 @@ r = self.request("POST","/collection",board_xml,headers) result = [] if r.status == 200: - data = r.read() + try: + data = r.read() + except socket.error, httplib.IncompleteRead: + return result # ËܲÈo2on ¤Î bug Âкö if data.rfind("") == -1: index = data.rfind("") @@ -330,9 +339,13 @@ def findvalue(self, kid): headers = {"X-O2-Target-Key":hexlify(kid)} r = self.request("GET","/findvalue",None,headers) + try: + data = r.read() + except socket.error, httplib.IncompleteRead: + return [] if r.status == 200: res = [] - dom = xml.dom.minidom.parseString(r.read()) + dom = xml.dom.minidom.parseString(data) nodes = dom.getElementsByTagName("nodes") keys = dom.getElementsByTagName("keys") if len(nodes): From o2on-svn ¡÷ lists.sourceforge.jp Fri Feb 19 11:22:48 2010 From: o2on-svn ¡÷ lists.sourceforge.jp (o2on svn commit) Date: Fri, 19 Feb 2010 11:22:48 +0900 Subject: [o2on-svn] [180] DatDB is now sqlite base. Message-ID: <1266546168.649620.23570.nullmailer@users.sourceforge.jp> Revision: 180 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=180 Author: nawota Date: 2010-02-19 11:22:48 +0900 (Fri, 19 Feb 2010) Log Message: ----------- DatDB is now sqlite base. Modified Paths: -------------- trunk/opy2on/lib/o2on_const.py trunk/opy2on/lib/o2on_dat.py trunk/opy2on/opy2on.py Modified: trunk/opy2on/lib/o2on_const.py =================================================================== --- trunk/opy2on/lib/o2on_const.py 2010-02-16 13:15:15 UTC (rev 179) +++ trunk/opy2on/lib/o2on_const.py 2010-02-19 02:22:48 UTC (rev 180) @@ -6,7 +6,7 @@ DBDir = "db" DatQueryFile = os.path.join(DBDir, "datq.pkl") KeyQueryFile = os.path.join(DBDir, "keyq.pkl") -DatDBFile = os.path.join(DBDir, "datdb.pkl") +DatDBFile = os.path.join(DBDir, "datdb.db") KeyDBFile = os.path.join(DBDir, "keydb.pkl") NodeDBFile = os.path.join(DBDir, "nodedb.pkl") ProfileFile = os.path.join(DBDir, "profile.pkl") Modified: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py 2010-02-16 13:15:15 UTC (rev 179) +++ trunk/opy2on/lib/o2on_dat.py 2010-02-19 02:22:48 UTC (rev 180) @@ -9,6 +9,8 @@ import gzip import random import time +import sqlite3 +from binascii import unhexlify, hexlify from o2on_const import regHosts, DatQueryFile, DatDBFile import o2on_config @@ -159,16 +161,15 @@ self.glob = g self.lock = threading.Lock() with self.lock: - self.hashmap = {} - self.boardmap = {} - self.publishmap = {} self.need_rebuild = False - self.load() - if len(self.hashmap) == 0: + if len(self)==0: self.need_rebuild = True def __len__(self): + if(not os.path.isfile(DatDBFile)): return 0 with self.lock: - return len(self.hashmap) + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.execute('SELECT COUNT(*) FROM dattbl') + return c.fetchone()[0] def checkrebuild(self): with self.lock: tmp = self.need_rebuild @@ -179,90 +180,108 @@ self.glob.logger.log("DATDB","Generated DatDB") with self.lock: self.need_rebuild = False + def makeDat(self, col): + if col: + dat = Dat(col[0]) + dat.published = col[1] + return dat + return None def getRandomInBoard(self,board): - if board in self.boardmap: - h = random.choice(self.boardmap[board]) - return self.hashmap[h] - return None + with self.lock: + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.execute('SELECT datpath, published FROM dattbl WHERE board = ? ORDER BY RANDOM() LIMIT 1', (board,)) + return self.makeDat(c.fetchone()) def choice(self): - return self.hashmap[random.choice(self.hashmap.keys())] + with self.lock: + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.execute('SELECT datpath,published FROM dattbl ORDER BY RANDOM() LIMIT 1') + return self.makeDat(c.fetchone()) def get(self,x): with self.lock: - return self.hashmap.get(x) + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.execute('SELECT datpath, published FROM dattbl WHERE hash = ?', (hexlify(x),)) + return self.makeDat(c.fetchone()) def has_keyhash(self,key): with self.lock: - return key in self.hashmap + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.execute('SELECT COUNT(*) FROM dattbl WHERE hash = ?', (key,)) + return c.fetchone()[0]==1 def add_dat(self, dat): with self.lock: - befdat = self.hashmap.get(dat.hash()) - self.hashmap[dat.hash()] = dat - if not dat.fullboard() in self.boardmap: - self.boardmap[dat.fullboard()] = [] - self.boardmap[dat.fullboard()].append(dat.hash()) - if not befdat: - dat.published = int(time.time()) - if dat.published not in self.publishmap: - self.publishmap[dat.published]=[] - self.publishmap[dat.published].append(dat.hash()) - else: - dat.published = befdat.published - def add(self, path, data, start=0): + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.cursor() + c.execute('SELECT datpath, published FROM dattbl WHERE hash = ?', + (hexlify(dat.hash()),)) + befdat = self.makeDat(c.fetchone()) + if not befdat: dat.published = int(time.time()) + else: dat.published = befdat.published + c.execute('REPLACE INTO dattbl VALUES(?, ?, ?, ?)', + (dat.path(), hexlify(dat.hash()), dat.fullboard(), dat.published)) + try: c.execute('COMMIT') + except sqlite3.OperationalError: pass + def add(self, path, data, start=0): dat = Dat(path) if dat.save(data, start): self.add_dat(dat) def published(self, datid, publish_time): if len(datid) != 20: raise Exception with self.lock: - if datid not in self.hashmap: raise Exception - dat = self.hashmap[datid] - self.publishmap[dat.published].remove(datid) + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.cursor() + c.execute('SELECT datpath, published FROM dattbl WHERE hash = ?', (hexlify(datid),)) + dat = self.makeDat(c.fetchone()) + if not dat: raise Exception dat.published = publish_time - if publish_time not in self.publishmap: self.publishmap[publish_time]=[] - self.publishmap[publish_time].append(datid) + c.execute('UPDATE dattbl SET published = ? WHERE hash = ?', (publish_time,hexlify(datid),)) + try: c.execute('COMMIT') + except sqlite3.OperationalError: pass def dat_to_publish(self, last_published_before, limit): res = [] if limit == 0: return res - for x in sorted(self.publishmap.keys()): - for y in self.publishmap[x]: - res.append(self.hashmap[y]) - limit -= 1 - if limit == 0: return res - return res + with self.lock: + sqlite_conn = sqlite3.connect(DatDBFile) + c=sqlite_conn.execute('SELECT datpath, published FROM dattbl '+ + 'WHERE published < ? ORDER BY published DESC LIMIT ?', + (last_published_before,limit)) + while True: + dat = self.makeDat(c.fetchone()) + if not dat: return res + res.append(dat) + return res def generate(self): + reghost = re.compile(regHosts+'$') + regnumdir = re.compile('^\d{4}$') regdat = re.compile('^(\d+)\.dat(?:\.gz)?$') - sep = re.escape(os.sep) - regdatdir = re.compile(regHosts+sep+'(.+)'+sep+'\d{4}$') with self.lock: - self.hashmap = {} - self.boardmap = {} - self.publishmap = {0:[]} - for root, dirs, files in os.walk(o2on_config.DatDir): - for f in files: - m1 = regdat.match(f) - if not m1: continue - m2 = regdatdir.search(root) - if not m2: continue - path = m2.group(1)+"/"+m2.group(2)+"/"+m1.group(1) - d = Dat(path) + sqlite_conn = sqlite3.connect(DatDBFile) + c = sqlite_conn.cursor() + try: c.execute('DROP TABLE dattbl') + except sqlite3.OperationalError: pass + c.execute('CREATE TABLE dattbl(datpath, hash PRIMARY KEY, board, published)') + for h in os.listdir(o2on_config.DatDir): + if not reghost.match(h): continue + for b in os.listdir(os.path.join(o2on_config.DatDir, h)): with self.lock: - self.hashmap[d.hash()] = d - if not d.fullboard() in self.boardmap: - self.boardmap[d.fullboard()] = [] - self.boardmap[d.fullboard()].append(d.hash()) - self.publishmap[0].append(d.hash()) - self.glob.logger.log("DATDB", "added %s" % path) + for d in os.listdir(os.path.join(o2on_config.DatDir, h, b)): + if not regnumdir.match(d): continue + for f in os.listdir(os.path.join(o2on_config.DatDir, h, b, d)): + m = regdat.match(f) + if not m: continue + path = h+"/"+b+"/"+m.group(1) + dat = Dat(path) + try: + c.execute('INSERT OR IGNORE INTO dattbl VALUES(?, ?, ?, ?)', + (path, hexlify(dat.hash()), dat.fullboard(), 0)) + except sqlite3.IntegrityError: + raise Exception("dup hash %s %s" % (hexlify(dat.hash()), path)) + self.glob.logger.log("DATDB", "added %s" % path) + try: c.execute('COMMIT') + except sqlite3.OperationalError: pass def load(self): - if(os.path.isfile(DatDBFile)): - pkl_file = open(DatDBFile,"rb") - with self.lock: - self.hashmap = cPickle.load(pkl_file) - self.boardmap = cPickle.load(pkl_file) - self.publishmap = cPickle.load(pkl_file) - pkl_file.close() + pass def save(self): - pkl_file = open(DatDBFile,"wb") with self.lock: - cPickle.dump(self.hashmap, pkl_file,-1) - cPickle.dump(self.boardmap, pkl_file,-1) - cPickle.dump(self.publishmap, pkl_file,-1) - pkl_file.close() + sqlite_conn = sqlite3.connect(DatDBFile) + try: sqlite_conn.execute('COMMIT') + except sqlite3.OperationalError: pass + Modified: trunk/opy2on/opy2on.py =================================================================== --- trunk/opy2on/opy2on.py 2010-02-16 13:15:15 UTC (rev 179) +++ trunk/opy2on/opy2on.py 2010-02-19 02:22:48 UTC (rev 180) @@ -25,11 +25,12 @@ import o2on_im def showstat(args): + n = (len(glob.nodedb), len(glob.datquery), len(glob.datdb), len(glob.keydb)) glob.logger.begin() - glob.logger.log("GLOBAL", "nodes %d" % len(glob.nodedb)) - glob.logger.log("GLOBAL", "datquery %d" % len(glob.datquery)) - glob.logger.log("GLOBAL", "dat %d" % len(glob.datdb)) - glob.logger.log("GLOBAL", "key %d" % len(glob.keydb)) + glob.logger.log("GLOBAL", "nodes %d" % n[0]) + glob.logger.log("GLOBAL", "datquery %d" % n[1]) + glob.logger.log("GLOBAL", "dat %d" % n[2]) + glob.logger.log("GLOBAL", "key %d" % n[3]) glob.logger.end() def showmynode(args): From o2on-svn ¡÷ lists.sourceforge.jp Sat Feb 20 21:44:40 2010 From: o2on-svn ¡÷ lists.sourceforge.jp (o2on svn commit) Date: Sat, 20 Feb 2010 21:44:40 +0900 Subject: [o2on-svn] [181] Hexlify key Message-ID: <1266669880.045643.16647.nullmailer@users.sourceforge.jp> Revision: 181 http://sourceforge.jp/projects/o2on/svn/view?view=rev&revision=181 Author: nawota Date: 2010-02-20 21:44:39 +0900 (Sat, 20 Feb 2010) Log Message: ----------- Hexlify key Modified Paths: -------------- trunk/opy2on/lib/o2on_dat.py Modified: trunk/opy2on/lib/o2on_dat.py =================================================================== --- trunk/opy2on/lib/o2on_dat.py 2010-02-19 02:22:48 UTC (rev 180) +++ trunk/opy2on/lib/o2on_dat.py 2010-02-20 12:44:39 UTC (rev 181) @@ -204,7 +204,7 @@ def has_keyhash(self,key): with self.lock: sqlite_conn = sqlite3.connect(DatDBFile) - c = sqlite_conn.execute('SELECT COUNT(*) FROM dattbl WHERE hash = ?', (key,)) + c = sqlite_conn.execute('SELECT COUNT(*) FROM dattbl WHERE hash = ?', (hexlify(key),)) return c.fetchone()[0]==1 def add_dat(self, dat): with self.lock: