first commit

Author: treper

algoroc.py

@@ -0,0 +1,21 @@
+import os,sys,glob,re
+if __name__=="__main__":
+    pf=open("positiveset.txt","r")
+    l=[]
+    for i in pf.readlines():
+        it=i.strip("\n")
+        l.append(it)
+        
+    pf.close()
+    rf=open("positive_result_to_choose.txt","r")
+    c=0
+    t=0
+    for i in rf.readlines():
+        it=i.strip("\n")
+        t=t+1
+        if it in l:
+            c=c+1
+    
+    rf.close()
+    print "FNR=",(1-float(c)/float(t))*100
+    rf=open("")

allocatekeys2slave.py

@@ -0,0 +1,59 @@
+import os,sys
+from subprocess import Popen, PIPE
+if __name__=="__main__":
+    argv=sys.argv
+    totalkeyslst=argv[1]
+    print totalkeyslst
+    slavelst=argv[2]
+    print slavelst
+    sl=open(slavelst,"r")
+    ips=[]
+    smap=[]
+    alkmap=[]
+    totalmem=0
+    ts=0
+    for i in sl.readlines():
+        line=i.strip("\n")
+        ip,mem=line.split(" ")
+        ips.append(ip)
+        smap.append(mem)
+        totalmem=totalmem+int(mem)
+        ts=ts+1
+    
+    tk=len(open(totalkeyslst,"r").readlines())  
+    tmp=0
+    for i in range(ts):
+        t=int(tk*int(smap[i])/totalmem)
+        alkmap.append(t)
+        tmp=tmp+t
+        #print smap[i],alkmap[i]
+    
+    alkmap[i]=tk-tmp+t
+
+    for i in alkmap:
+        print i
+    onlst=[]
+    d,ext=os.path.splitext(totalkeyslst)
+    for i in range(ts):
+        onlst.append(d+str(i+1)+ext)
+    
+    j=0
+    ct=0
+    for i in range(ts):
+        if j==0:
+            cmd="head -"+str(alkmap[i])+" "+totalkeyslst+" >"+onlst[j]
+        else:
+            cmd="head -"+str(ct+alkmap[i])+" "+totalkeyslst+"|tail -"+str(alkmap[i])+" >"+onlst[j]
+        os.system(cmd)
+        print cmd
+        if j==0:
+            pass
+        else:
+            cmd="scp "+onlst[j]+" is_admin@"+ips[j]+":"+totalkeyslst
+            os.system(cmd)
+        ct=ct+alkmap[i]
+        j=j+1
+
+    
+    
+    

cp2fingertest.py

@@ -0,0 +1,15 @@
+import os,sys,glob,time
+from subprocess import Popen, PIPE
+dest=" [email protected]:/home/is_admin/videos/negative/"
+while(True):
+    l=glob.glob("*.f4v")
+    for i in l:
+        p1 = Popen(['fuser',i], stdout=PIPE)
+        if p1.communicate()[0]=='':
+            cmd="scp "+i+dest
+            os.system(cmd)
+            cmd="rm -f "+i
+            os.system(cmd)
+    
+    time.sleep(60)
+        

distriKeys.py

@@ -0,0 +1,21 @@
+from hash_ring import *
+memcache_servers = ['127.0.0.1','10.25.13.152','10.25.13.154']
+ring = HashRing(memcache_servers)
+f=open("keyslist-all.txt","r")
+fo=open("distrikeys.txt","w")
+d={}
+for i in memcache_servers:
+    d[i]=0
+    
+for i in f.readlines():
+    itemId=i.strip("\n")
+    server = ring.get_node(itemId)
+    d[server]=d[server]+1
+    #print server
+    fo.write(itemId+" "+server+"\n")
+f.close()
+fo.close()
+
+for i in memcache_servers:
+    print d[i]
+    

downVideo.py

@@ -0,0 +1,53 @@
+import urllib,cookielib,re,os
+from xml.etree.ElementTree import ElementTree,fromstring,tostring
+def getVideo(id,done):
+	id=id.strip('\n')
+	#video_name="/home/is_admin/videos/"+id+".f4v"
+	video_name="E:/newvideos/"+id+".f4v"
+	#print video_name 
+	#return
+	if not os.path.exists(video_name) and id not in done:
+		url="http://v2.tudou.com/v?it="+id
+		cj=cookielib.CookieJar()
+		r=urllib.urlopen(url)
+		html=r.read()
+		#print html
+		pr=re.compile("http://(?:.|\n)*?</f>")
+		ma=re.search(pr,html)
+		if(ma==None):
+			return
+		myxml=fromstring(html)
+		#try:
+		#	title=myxml.get('title')
+		#except:
+		#	return
+		#video_url=myxml[0][0].text
+		video_url=ma.group(0)[:-4]
+		#print video_url
+		
+		#video_name=id+".f4v"
+		#fo.write(id+" "+title.encode("utf-8")+"\n")
+		#print id,title
+		#print video_name
+		try:
+			urllib.urlretrieve(video_url,video_name)
+		except:
+			return
+
+if __name__=='__main__':
+	f=open("keyslist-all.txt")
+	fo=open("D:/videos/downloaded.txt","r")
+	done=[]
+	for i in fo.readlines():
+		it=i.strip("\n")
+		done.append(it)
+	fo.close()
+	fo=open("D:/refer/refer.txt","r")
+	for i in fo.readlines():
+		it=i.strip("\n")
+		done.append(it)
+	fo.close()
+	for i in f.readlines():
+		#print i
+		getVideo(i,done)
+	f.close()

genapp.py

@@ -0,0 +1,36 @@
+import os,sys,glob,time,json
+slavelst=['127.0.0.1','10.25.13.152','10.25.13.153','10.25.13.154']
+loadkeydir={}
+loadkeydir['VF12']='/home/is_admin/MRPlatform/src/Test/keys/keyslist-all.txt'
+testkeydir['VF6']='/home/is_admin/videos/'
+loadkeydir['AF']='/home/is_admin/MRPlatform/src/Test/keys/keyslist-all.txt'
+method=['VF12','VF6','AF']
+min_score=range(5,26)
+seg=[100,200,400,600,800,1000,1200,1400,1600]
+neighborhood=range(1,3)
+f=open("./config/cfg.lst","r")
+for i in f.readlines():
+    cfg_list.append(i.strip("\n"))
+for i in cfg_list:
+    #change app config files and switch parameters
+    pmaster.stdin.write("")
+    #generate
+    lt=0
+    lr=1
+    while(lt!=lr):
+        #get the tested keys number from appconfig file i
+        json_data = open(i)
+        data = json.load()
+        loadlist = str(data["KeysLocation"]["LoadKeysList"])
+        print loadlist
+        #%sms%2.2fseg%dsample%dcutoff%2.2fv%dn%d%s_result.txt
+        pon = os.path.basename(str(data["KeysLocation"]["TestKeysList"]))[0:8]
+        of=str(data["AlgoParams"]["method"])+"ms"+'%2.2f'%float(str(data["AlgoParams"]["em_min_score"]))+"seg"+str(data["AlgoParams"]["keys_num_a_seg"])+"sample"+str(data["AlgoParams"]["samplingNum"])+"cutoff"+str(data["AlgoParams"]["cutoffpercent"])+"v"+str(data["AlgoParams"]["verifymethodid"])+"n"+str(data["AlgoParams"]["neighbormethodid"])+pon+"_result.txt"
+        print of
+        pc = Popen("cat "+ loadlist + " |wc -l", stdin = PIPE, stdout = PIPE, stderr = None, shell = True)
+        lt = int(pc.stdout.read())
+        print lt
+        pc = Popen("cat "+ of + " |wc -l", stdin = PIPE, stdout = PIPE, stderr = None, shell = True)
+        lr = int(pc.stdout.read())
+        print lr
+        time.sleep(60)

getTilelist.py

@@ -0,0 +1,40 @@
+import urllib,cookielib,re,os
+from xml.etree.ElementTree import ElementTree,fromstring,tostring
+def getVideo(fo,id):
+	id=id.strip('\n')
+	video_name="E:/videos/negative/"+id+".f4v"
+	if not os.path.exists(video_name):
+		url="http://v2.tudou.com/v?it="+id
+		cj=cookielib.CookieJar()
+		r=urllib.urlopen(url)
+		html=r.read()
+		#print html
+		pr=re.compile("http://(?:.|\n)*?</f>")
+		ma=re.search(pr,html)
+		if(ma==None):
+			return
+		myxml=fromstring(html)
+		try:
+			title=myxml.get('title')
+		except:
+			return
+		#video_url=myxml[0][0].text
+		video_url=ma.group(0)[:-4]
+		#print video_url
+		
+		#video_name=id+".f4v"
+		fo.write(id+" "+title.encode("utf-8")+"\n")
+		print id,title
+		#print video_name
+		#try:
+			#urllib.urlretrieve(video_url,video_name)
+		#except:
+			#return
+
+if __name__=='__main__':
+	f=open("E:/false_positive.txt")
+	fo=open("E:/false_positive_title.txt","w")
+	for i in f.readlines():
+		#print i
+		getVideo(fo,i)
+	fo.close()

gettudouv.py

@@ -0,0 +1,42 @@
+import urllib,cookielib,re,os
+import os.path
+from xml.etree.ElementTree import ElementTree,fromstring,tostring
+
+def getVideo(id,video_name):
+	id=id.strip('\n')
+	#video_name="E:/videos/negative/"+id+".f4v"
+	if not os.path.exists(video_name):
+		url="http://v2.tudou.com/v?it="+id
+		cj=cookielib.CookieJar()
+		r=urllib.urlopen(url)
+		html=r.read()
+		#print html
+		pr=re.compile("http://(?:.|\n)*?</f>")
+		ma=re.search(pr,html)
+		if(ma==None):
+			return
+		myxml=fromstring(html)
+		try:
+			title=myxml.get('title')
+		except:
+			return
+		#video_url=myxml[0][0].text
+		video_url=ma.group(0)[:-4]
+		#print video_url
+		
+		#video_name=id+".f4v"
+		#fo.write(id+" "+title.encode("utf-8")+"\n")
+		#print id,title
+		#print video_name
+		try:
+			urllib.urlretrieve(video_url,video_name)
+		except:
+			return
+
+if __name__=='__main__':
+	for pa,dn,fn in os.walk("E:/videos/positive"):
+		#print pa,pa[:20]
+		for j in dn:
+			itemId=j[:9]
+			name=pa+"/"+j+"/"+itemId+"tudou.f4v"
+			getVideo(itemId,name)

getvideolist.py

@@ -0,0 +1,58 @@
+from BeautifulSoup import BeautifulSoup
+import mechanize,re,urllib
+def getVideo(sw,fo):
+    br = mechanize.Browser()
+    br.set_handle_robots(False)
+    #br.open("http://cn.bing.com/?scope=video&FORM=Z9LH1")
+    #br.select_form(nr=0)
+    #br.form['q']="Girls Generation"
+    #rb=br.open("http://cn.bing.com/videos/search?pq=girls+generation&sc=8-16&sp=-1&sk=&qpvt=Girls+Generation&q=Girls+Generation%20filterui:msite-youku.com&FORM=R5FD8")
+    #http://cn.bing.com/videos/search?pq=girls+generation&sc=8-16&sp=-1&sk=&qpvt=Girls+Generation&q=Girls+Generation%20filterui:msite-youku.com&FORM=R5FD9
+    #rb=br.open("http://cn.bing.com/videos/search?pq=girls+generation&sc=8-16&sp=-1&sk=&qpvt=Girls+Generation&q=Girls+Generation%20filterui:msite-ku6.com&FORM=R5FD1")
+    #http://cn.bing.com/videos/search?pq=girls+generation&sc=8-16&sp=-1&sk=&qpvt=Girls+Generation&q=Girls+Generation%20filterui:msite-6.cn&FORM=R5FD12
+    #rb=br.submit()
+    #vsite=['youku','ku6']
+    #replace the space in search word
+    #sw
+    #for site in vsite:
+        #surl="http://cn.bing.com/videos/search?pq=%s&sc=8-16&sp=-1&sk=&qpvt=Girls+Generation&q=Girls+Generation%20filterui:msite-%s.com&FORM=R5FD8"%site
+        #rb=br.open(surl)
+        #http://cn.bing.com/videos/search?q=%E9%92%A2%E9%93%81%E4%BE%A0&go=&qs=n&form=QBLH&pq=%E9%92%A2%E9%93%81%E4%BE%A0&sc=0-4&sp=-1&sk=
+    #sw=sw.replace(" ","+")
+    #sw=sw.decode("utf-8")
+    #sw=sw.encode("gb2312")
+    sw=urllib.quote(sw)
+    longtime=urllib.quote(" filterui:duration-long")
+    #rb=br.open("http://cn.bing.com/videos/search?q=%E8%80%81%E5%A4%A7%E7%9A%84%E5%B9%B8%E7%A6%8F34&go=&qs=n&form=QBLH&pq=%E8%80%81%E5%A4%A7%E7%9A%84%E5%B9%B8%E7%A6%8F34&sc=1-7&sp=-1&sk=")
+    #http://cn.bing.com/videos/search?pq=%u65b9%u8c2c%u795e%u63a206&sc=0-0&sp=-1&sk=&qpvt=%E6%96%B9%E8%B0%AC%E7%A5%9E%E6%8E%A206&q=%E6%96%B9%E8%B0%AC%E7%A5%9E%E6%8E%A206%20filterui:duration-long&FORM=R5FD2
+    surl="http://cn.bing.com/videos/search?q=%(sw)s&go=&qs=n&form=Z9LH1&pq=%(sw)s%(longtime)s&sc=1-7&sp=-1&sk="%{"sw":sw,"longtime":longtime}
+    print surl
+    rb=br.open(surl)
+    content = rb.read()
+    vr=re.compile("purl=\"http://(?:.|\n)*?\"")
+    #vr=re.compile("result-link\"\ href=\"http://(?:.|\n)*?\"")
+    #vr=re.compile("result-link")
+    #vr['youku']=re.compile("http://v.youku.com/v_show/id_(?:.|\n)*?.html")
+    #vr['ku6']=re.compile("http://v.ku6.com/show/(?:.|\n)*?.html")
+    ma=re.findall(vr,content)
+    print len(ma)
+    for i in ma:
+        if i.find("tudou")==-1 and i.find("cntv")==-1 and i.find("xinhuanet")==-1:
+            print i[6:-1]
+            fo.write(i[6:-1]+"\n")
+
+def main():
+    f=open("idtilte2.lst")
+    fo=open("videopage2.lst","w")
+    for sw in f.readlines():
+        l=sw.find(" ")
+        title=sw[l:]
+        getVideo(title,fo)
+    fo.close()
+    
+if __name__=="__main__":
+    main()   
+
+#import os
+#os.chdir("E:/videos")
+#from kw2vlBaidu import *

jujilist.py

@@ -0,0 +1,15 @@
+import re,urllib
+f=open("id-title.txt","rb")
+fo=open("juji.lst","w")
+pr=re.compile("[\u2e80-\uffff]+\d\d")
+for i in f.readlines():
+    l=i.find(" ")
+    k=i[:l]
+    title = i[l:]
+    ma=re.findall(pr,title)
+    if ma and len(title)<10:
+        #print title
+        fo.write(k+" "+title+"\n")
+
+f.close()
+fo.close()

keygen.py

@@ -0,0 +1,16 @@
+import os,sys,re
+
+FFMPEG="ffmpeg"
+CODEWAV="/root/tmp/musicretr-2.2.6/codewav"
+f=open("video_list.lst","r")
+for vn in f.readlines():
+    inputfn=vn.strip("\n")
+    outputfn=os.path.splitext(vn)[0]+".key"
+    cmd = FFMPEG+" -i "+inputfn+" -y  -ar 11025 -ab 128k tmp.wav"
+    os.system(cmd)
+    #print cmd
+    cmd = CODEWAV+" boostextdescr.txt 1 tmp.wav "+outputfn
+    #print cmd
+    os.system(cmd)
+    
+f.close()