iotcat
/
PyOne
mirror of https://github.com/IoTcat/PyOne.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

填坑

Browse Source
master
abbeyokgo 5 years ago
parent 1db5aadf61
commit d2611fdcee
10 changed files with 214 additions and 265 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      app/__init__.py
  2. 72
      app/admin/views.py
  3. 4
      app/extend.py
  4. 37
      app/utils/common.py
  5. 83
      app/utils/file_os.py
  6. 98
      app/utils/header.py
  7. 74
      app/utils/offdownload.py
  8. 106
      app/utils/updatefile.py
  9. 2
      app/utils/upload.py
  10. 2
      config.py

1
app/__init__.py
Unescape View File

@ -11,7 +11,6 @@ def create_app():
config.init_app(app)
app.wsgi_app = ProxyFix(app.wsgi_app)
cache.init_app(app)
redis_client.init_app(app)
limiter.init_app(app)
from .front import front as front_blueprint

72
app/admin/views.py
Unescape View File

@ -250,39 +250,7 @@ def edit():
fileid=request.form.get('fileid')
user=request.form.get('user')
content=request.form.get('content').encode('utf-8')
info={}
token=GetToken(user=user)
app_url=GetAppUrl()
headers={'Authorization':'bearer {}'.format(token)}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/items/{}/content'.format(fileid)
try:
r=requests.put(url,headers=headers,data=content,timeout=10)
data=json.loads(r.content)
if data.get('id'):
info['status']=0
info['msg']='修改成功'
redis_client.delete('{}:content'.format(fileid))
file=mongo.db.items.find_one({'id':fileid})
name=file['name']
path=file['path'].replace(name,'',1)
if len(path.split('/'))>2 and path.split('/')[-1]=='':
path=path[:-1]
# if path=='':
# path='/'
# if not path.startswith('/'):
# path='/'+path
# path='{}:{}'.format(user,path)
key='has_item$#$#$#$#{}$#$#$#$#{}'.format(path,name)
print('edit key:{}'.format(key))
redis_client.delete(key)
else:
info['status']=0
info['msg']=data.get('error').get('message')
except Exception as e:
print e
info['status']=0
info['msg']='修改超时'
info=EditFile(fileid=fileid,content=content,user=user)
return jsonify(info)
fileid=request.args.get('fileid')
user=request.args.get('user')
@ -310,7 +278,6 @@ def upload_local():
@admin.route('/checkChunk', methods=['POST'])
def checkChunk():
GetToken=request.form.get('fil GetToken')
fileName=request.form.get('name').encode('utf-8')
chunk=request.form.get('chunk',0,type=int)
filename = u'./upload/{}-{}'.format(fileName, chunk)
@ -386,33 +353,8 @@ def setFile(filename=None):
path=path.split(':')[0]+':/'
user,n_path=path.split(':')
filename=request.form.get('filename')
if not n_path.startswith('/'):
n_path='/'+n_path
share_path=od_users.get(user).get('share_path')
if share_path!='/':
remote_file=os.path.join(os.path.join(share_path,n_path[1:]),filename)
else:
remote_file=os.path.join(n_path,filename)
print(u'remote path:{}'.format(remote_file))
content=request.form.get('content').encode('utf-8')
info={}
token=GetToken(user=user)
app_url=GetAppUrl()
headers={'Authorization':'bearer {}'.format(token)}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/items/root:{}:/content'.format(remote_file)
r=requests.put(url,headers=headers,data=content,timeout=10)
data=json.loads(r.content)
if data.get('id'):
AddResource(data,user)
info['status']=0
info['msg']='添加成功'
key='has_item$#$#$#$#{}$#$#$#$#{}'.format(path,filename)
print('set key:{}'.format(key))
redis_client.delete(key)
else:
info['status']=0
info['msg']=data.get('error').get('message')
info=CreateFile(filename=filename,path=n_path,content=content,user=user)
return jsonify(info)
path=urllib.unquote(request.args.get('path'))
if path.split(':')[-1]=='':
@ -437,7 +379,7 @@ def delete():
infos['fail']=0
for id in ids:
print 'delete {}'.format(id)
file=mongo.db.items.find_one({'id':id})
file=mon_db.items.find_one({'id':id})
name=file['name']
path=file['path'].replace(name,'')
if len(path.split('/'))>2 and path.split('/')[-1]=='':
@ -540,7 +482,7 @@ def RPCserver():
@admin.route('/clearHist',methods=['POST'])
def clearHist():
mongo.db.down_db.delete_many({})
mon_db.down_db.delete_many({})
ret={'msg':'清除成功！'}
return jsonify(ret)
@ -642,8 +584,8 @@ def install():
def uninstall():
type_=request.form.get('type')
if type_=='mongodb':
mongo.db.items.remove()
mongo.db.down_db.remove()
mon_db.items.remove()
mon_db.down_db.remove()
msg='删除mongodb数据成功'
elif type_=='redis':
redis_client.flushdb()
@ -731,7 +673,7 @@ def rm_pan():
f.write(new_text)
key='users'
redis_client.delete(key)
mongo.db.items.delete_many({'user':pan})
mon_db.items.delete_many({'user':pan})
data=dict(msg='删除盘符[{}]成功'.format(pan),status=1)
return jsonify(data)
return render_template('admin/pan_manage/rm_pan.html')

4
app/extend.py
Unescape View File

@ -1,12 +1,8 @@
#-*- coding=utf-8 -*-
from flask_caching import Cache
from flask_pymongo import PyMongo
from flask_redis import Redis
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
cache = Cache()
mongo = PyMongo()
redis_client=Redis()
limiter = Limiter(key_func=get_remote_address)

37
app/utils/common.py
Unescape View File

@ -19,11 +19,11 @@ def GetTotal(path='A:/'):
else:
user,n_path=path.split(':')
if n_path=='/':
total=mongo.db.items.find({'grandid':0}).count()
total=mon_db.items.find({'grandid':0}).count()
else:
f=mongo.db.items.find_one({'path':path})
f=mon_db.items.find_one({'path':path})
pid=f['id']
total=mongo.db.items.find({'parent':pid}).count()
total=mon_db.items.find({'parent':pid}).count()
redis_client.set(key,total,300)
return total
@ -32,7 +32,7 @@ def GetTotal(path='A:/'):
def FetchData(path='A:/',page=1,per_page=50,sortby='lastModtime',order='desc',dismiss=False,search_mode=False):
if search_mode:
show_secret=GetConfig('show_secret')
query=mongo.db.items.find({'name':re.compile(path)})
query=mon_db.items.find({'name':re.compile(path)})
resp=[]
data=query.limit(per_page).collation({"locale": "zh", 'numericOrdering':True})\
.sort([('order',ASCENDING)])\
@ -73,7 +73,7 @@ def FetchData(path='A:/',page=1,per_page=50,sortby='lastModtime',order='desc',di
try:
user,n_path=path.split(':')
if n_path=='/':
data=mongo.db.items.find({'grandid':0,'user':user}).collation({"locale": "zh", 'numericOrdering':True})\
data=mon_db.items.find({'grandid':0,'user':user}).collation({"locale": "zh", 'numericOrdering':True})\
.sort([('order',ASCENDING),(sortby,order)])\
.limit(per_page).skip((page-1)*per_page)
for d in data:
@ -91,11 +91,11 @@ def FetchData(path='A:/',page=1,per_page=50,sortby='lastModtime',order='desc',di
resp.append(item)
total=GetTotal(path)
else:
f=mongo.db.items.find_one({'path':path})
f=mon_db.items.find_one({'path':path})
pid=f['id']
if f['type']!='folder':
return f,'files'
data=mongo.db.items.find({'parent':pid}).collation({"locale": "zh", 'numericOrdering':True})\
data=mon_db.items.find({'parent':pid}).collation({"locale": "zh", 'numericOrdering':True})\
.sort([('order',ASCENDING),(sortby,order)])\
.limit(per_page).skip((page-1)*per_page)
for d in data:
@ -255,13 +255,18 @@ def file_ico(item):
def _remote_content(fileid,user):
kc='{}:content'.format(fileid)
if redis_client.exists(kc):
return redis_client.get(kc)
content=unicode(redis_client.get(kc), errors='ignore')
return content
else:
downloadUrl,play_url=GetDownloadUrl(fileid,user)
if downloadUrl:
r=requests.get(downloadUrl)
# r.encoding='utf-8'
content=r.text
if r.encoding=='ISO-8859-1':
content=r.content
else:
content=r.text
print(content)
redis_client.set(kc,content)
return content
else:
@ -269,7 +274,7 @@ def _remote_content(fileid,user):
# @cache.memoize(timeout=60)
def has_item(path,name):
if mongo.db.items.count()==0:
if mon_db.items.count()==0:
return False,False,False
key='has_item$#$#$#$#{}$#$#$#$#{}'.format(path,name)
print('get key:{}'.format(key))
@ -295,26 +300,26 @@ def has_item(path,name):
try:
user,n_path=path.split(':')
if n_path=='/':
if mongo.db.items.find_one({'grandid':0,'name':name,'user':user}):
fid=mongo.db.items.find_one({'grandid':0,'name':name,'user':user})['id']
if mon_db.items.find_one({'grandid':0,'name':name,'user':user}):
fid=mon_db.items.find_one({'grandid':0,'name':name,'user':user})['id']
item=_remote_content(fid,user).strip()
else:
route=n_path[1:].split('/')
if name=='.password':
for idx,r in enumerate(route):
p=user+':/'+'/'.join(route[:idx+1])
f=mongo.db.items.find_one({'path':p})
f=mon_db.items.find_one({'path':p})
pid=f['id']
data=mongo.db.items.find_one({'name':name,'parent':pid})
data=mon_db.items.find_one({'name':name,'parent':pid})
if data:
fid=data['id']
item=_remote_content(fid,user).strip()
if idx==len(route)-1:
cur=True
else:
f=mongo.db.items.find_one({'path':path})
f=mon_db.items.find_one({'path':path})
pid=f['id']
data=mongo.db.items.find_one({'name':name,'parent':pid})
data=mon_db.items.find_one({'name':name,'parent':pid})
if data:
fid=data['id']
item=_remote_content(fid,user).strip()

83
app/utils/file_os.py
Unescape View File

@ -3,7 +3,7 @@ from header import *
########################删除文件
def DeleteLocalFile(fileid):
mongo.db.items.remove({'id':fileid})
mon_db.items.remove({'id':fileid})
def DeleteRemoteFile(fileid,user='A'):
app_url=GetAppUrl()
@ -29,7 +29,7 @@ def CreateFolder(folder_name,grand_path,user='A'):
grandid=0
else:
path='{}:/{}'.format(user,grand_path)
parent=mongo.db.mongo.db.items.find_one({'path':path})
parent=mon_db.items.find_one({'path':path})
parent_id=parent['id']
grandid=parent['grandid']+1
url=app_url+'v1.0/me/drive/items/{}/children'.format(parent['id'])
@ -64,12 +64,73 @@ def CreateFolder(folder_name,grand_path,user='A'):
path='{}:{}'.format(user,path)
item['path']=path
item['order']=0
mongo.db.items.insert_one(item)
mon_db.items.insert_one(item)
return True
else:
print(data.get('error').get('msg'))
return False
def CreateFile(filename,path,content,user='A'):
token=GetToken(user=user)
app_url=GetAppUrl()
if not path.startswith('/'):
path='/'+path
share_path=od_users.get(user).get('share_path')
if share_path!='/':
remote_file=os.path.join(os.path.join(share_path,path[1:]),filename)
else:
remote_file=os.path.join(path,filename)
print(u'remote path:{}'.format(remote_file))
info={}
headers={'Authorization':'bearer {}'.format(token)}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/items/root:{}:/content'.format(remote_file)
r=requests.put(url,headers=headers,data=content,timeout=10)
data=json.loads(r.content)
if data.get('id'):
AddResource(data,user)
info['status']=0
info['msg']='添加成功'
key='has_item$#$#$#$#{}:{}$#$#$#$#{}'.format(user,path,filename)
print('set key:{}'.format(key))
redis_client.delete(key)
else:
info['status']=0
info['msg']=data.get('error').get('message')
return info
def EditFile(fileid,content,user='A'):
token=GetToken(user=user)
app_url=GetAppUrl()
info={}
headers={'Authorization':'bearer {}'.format(token)}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/items/{}/content'.format(fileid)
try:
r=requests.put(url,headers=headers,data=content,timeout=10)
data=json.loads(r.content)
if data.get('id'):
info['status']=0
info['msg']='修改成功'
redis_client.delete('{}:content'.format(fileid))
file=mon_db.items.find_one({'id':fileid})
name=file['name']
path=file['path'].replace(name,'',1)
if len(path.split('/'))>2 and path.split('/')[-1]=='':
path=path[:-1]
key='has_item$#$#$#$#{}$#$#$#$#{}'.format(path,name)
print('edit key:{}'.format(key))
redis_client.delete(key)
else:
info['status']=0
info['msg']=data.get('error').get('message')
except Exception as e:
print e
info['status']=0
info['msg']='修改超时'
return info
def MoveFile(fileid,new_folder_path,user='A'):
app_url=GetAppUrl()
token=GetToken(user=user)
@ -82,7 +143,7 @@ def MoveFile(fileid,new_folder_path,user='A'):
else:
path='{}:/{}'.format(user,new_folder_path)
print path
parent_item=mongo.db.items.find_one({'path':path})
parent_item=mon_db.items.find_one({'path':path})
folder_id=parent_item['id']
parent=parent_item['id']
grandid=parent_item['grandid']+1
@ -100,13 +161,13 @@ def MoveFile(fileid,new_folder_path,user='A'):
data=json.loads(r.content)
if data.get('id'):
new_value={'parent':parent,'grandid':grandid,'path':path}
mongo.db.items.find_one_and_update({'id':fileid},{'$set':new_value})
file=mongo.db.items.find_one({'id':fileid})
mon_db.items.find_one_and_update({'id':fileid},{'$set':new_value})
file=mon_db.items.find_one({'id':fileid})
filename=file['name']
if file['parent']=='':
path='/'
else:
path=mongo.db.items.find_one({'id':file['parent']})['path']
path=mon_db.items.find_one({'id':file['parent']})['path']
key='has_item$#$#$#$#{}$#$#$#$#{}'.format(path,filename)
redis_client.delete(key)
return True
@ -126,21 +187,21 @@ def ReName(fileid,new_name,user='A'):
r=requests.patch(url,headers=headers,data=json.dumps(payload))
data=json.loads(r.content)
if data.get('id'):
it=mongo.db.items.find_one({'id':fileid})
it=mon_db.items.find_one({'id':fileid})
old_name=it['name']
path=it['path'].replace(old_name,new_name,1)
new_value={'path':path,'name':new_name}
mongo.db.items.find_one_and_update({'id':fileid},{'$set':new_value})
mon_db.items.find_one_and_update({'id':fileid},{'$set':new_value})
key='path:{}'.format(fileid)
redis_client.delete(key)
key='name:{}'.format(fileid)
redis_client.delete(key)
if it['type']=='folder':
files=mongo.db.items.find({'parent':it['id']})
files=mon_db.items.find({'parent':it['id']})
for file in files:
new_path=file['path'].replace(old_name,new_name,1)
new_value={'path':new_path}
mongo.db.items.find_one_and_update({'id':file['id']},{'$set':new_value})
mon_db.items.find_one_and_update({'id':file['id']},{'$set':new_value})
return True
else:
print(data.get('error').get('msg'))

98
app/utils/header.py
Unescape View File

@ -21,15 +21,31 @@ import signal
from dateutil.parser import parse
from Queue import Queue
from threading import Thread,Event
import redis
from pymongo import uri_parser,MongoClient,ASCENDING,DESCENDING
from self_config import *
from aria2 import PyAria2
from ..extend import *
from config import config
#######Mongodb
mongo = MongoClient(config.MONGO_URI,**{'connect':False})
def GetConfig_pre(key):
if key=='allow_site':
value=','.join(allow_site)
else:
value=eval(key)
return value
#######Mongodb & redis
mongo = MongoClient(host=GetConfig_pre('MONGO_HOST'),port=int(GetConfig_pre('MONGO_PORT')),connect=False)
mon_db=eval('mongo.{}'.format(GetConfig_pre('MONGO_DB')))
if GetConfig_pre('MONGO_PASSWORD')!='':
mon_db.authenticate(GetConfig_pre('MONGO_USER'),GetConfig_pre('MONGO_PASSWORD'))
if GetConfig_pre('REDIS_PASSWORD')!='':
redis_client=redis.Redis(host=GetConfig_pre('REDIS_HOST'),port=int(GetConfig_pre('REDIS_PORT')),db=GetConfig_pre('REDIS_DB'),password=GetConfig_pre('REDIS_PASSWORD'))
else:
redis_client=redis.Redis(host=GetConfig_pre('REDIS_HOST'),port=int(GetConfig_pre('REDIS_PORT')),db=GetConfig_pre('REDIS_DB'))
#######授权链接
LoginUrl=BaseAuthUrl+'/common/oauth2/v2.0/authorize?response_type=code\
@ -62,7 +78,7 @@ def GetName(id):
if redis_client.exists(key):
return redis_client.get(key)
else:
item=mongo.db.items.find_one({'id':id})
item=mon_db.items.find_one({'id':id})
redis_client.set(key,item['name'])
return item['name']
@ -71,25 +87,10 @@ def GetPath(id):
if redis_client.exists(key):
return redis_client.get(key)
else:
item=mongo.db.items.find_one({'id':id})
item=mon_db.items.find_one({'id':id})
redis_client.set(key,item['path'])
return item['path']
def GetConfig(key):
if key=='allow_site':
value=redis_client.get('allow_site') if redis_client.exists('allow_site') else ','.join(allow_site)
else:
value=redis_client.get(key) if redis_client.exists(key) else eval(key)
#这里是为了储存
if key=='od_users' and isinstance(value,dict):
value=json.dumps(value)
if not redis_client.exists(key):
redis_client.set(key,value)
#这里是为了转为字典
if key=='od_users':
value=json.loads(value)
return value
def GetThemeList():
tlist=[]
theme_dir=os.path.join(config_dir,'app/templates/theme')
@ -173,7 +174,7 @@ def CheckTimeOut(fileid):
token=GetToken()
headers={'Authorization':'bearer {}'.format(token),'Content-Type':'application/json'}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/mongo.db.items/'+fileid
url=app_url+'v1.0/me/drive/mon_db.items/'+fileid
r=requests.get(url,headers=headers)
data=json.loads(r.content)
if data.get('@microsoft.graph.downloadUrl'):
@ -187,7 +188,7 @@ def CheckTimeOut(fileid):
def RemoveRepeatFile():
"""
db.mongo.db.items.aggregate([
db.mon_db.items.aggregate([
{
$group:{_id:{id:'$id'},count:{$sum:1},dups:{$addToSet:'$_id'}}
},
@ -198,11 +199,11 @@ def RemoveRepeatFile():
]).forEach(function(it){
it.dups.shift();
db.mongo.db.items.remove({_id: {$in: it.dups}});
db.mon_db.items.remove({_id: {$in: it.dups}});
});
"""
deleteData=mongo.db.items.aggregate([
deleteData=mon_db.items.aggregate([
{'$group': {
'_id': { 'id': "$id"},
'uniqueIds': { '$addToSet': "$_id" },
@ -218,7 +219,7 @@ def RemoveRepeatFile():
first=True
for did in d['uniqueIds']:
if not first:
mongo.db.items.delete_one({'_id':did});
mon_db.items.delete_one({'_id':did});
first=False
except Exception as e:
print(e)
@ -276,6 +277,21 @@ def _file_content(path,offset,length):
return content
def GetConfig(key):
if key=='allow_site':
value=redis_client.get('allow_site') if redis_client.exists('allow_site') else ','.join(allow_site)
else:
value=redis_client.get(key) if redis_client.exists(key) else eval(key)
#这里是为了储存
if key=='od_users' and isinstance(value,dict):
value=json.dumps(value)
if not redis_client.exists(key):
redis_client.set(key,value)
#这里是为了转为字典
if key=='od_users':
value=json.loads(value)
return value
def AddResource(data,user='A'):
#检查父文件夹是否在数据库，如果不在则获取添加
@ -298,7 +314,7 @@ def AddResource(data,user='A'):
parent_path='/'
pid=''
for idx,p in enumerate(grand_path.split('/')):
parent=mongo.db.items.find_one({'name':p,'grandid':idx,'parent':pid})
parent=mon_db.items.find_one({'name':p,'grandid':idx,'parent':pid})
if parent is not None:
pid=parent['id']
parent_path='/'.join([parent_path,parent['name']])
@ -319,7 +335,7 @@ def AddResource(data,user='A'):
item['grandid']=idx
item['parent']=pid
item['path']=path
mongo.db.items.insert_one(item)
mon_db.items.insert_one(item)
pid=fdata.get('id')
#插入数据
item={}
@ -352,14 +368,14 @@ def AddResource(data,user='A'):
item['order']=1
else:
item['order']=2
mongo.db.items.insert_one(item)
mon_db.items.insert_one(item)
def CheckTimeOut(fileid):
app_url=GetAppUrl()
token=GetToken()
headers={'Authorization':'bearer {}'.format(token),'Content-Type':'application/json'}
headers.update(default_headers)
url=app_url+'v1.0/me/drive/mongo.db.items/'+fileid
url=app_url+'v1.0/me/drive/mon_db.items/'+fileid
r=requests.get(url,headers=headers)
data=json.loads(r.content)
if data.get('@microsoft.graph.downloadUrl'):
@ -377,7 +393,6 @@ class GetItemThread(Thread):
super(GetItemThread,self).__init__()
self.queue=queue
self.user=user
self._stop_event = Event()
share_path=od_users.get(user).get('share_path')
if share_path=='/':
self.share_path=share_path
@ -399,15 +414,8 @@ class GetItemThread(Thread):
trytime=info['trytime']
self.GetItem(url,grandid,parent,trytime)
if self.queue.empty():
# print('{} queue empty!break'.format(self.user))
break
def stop(self):
self._stop_event.set()
def stopped(self):
return self._stop_event.is_set()
def GetItem(self,url,grandid=0,parent='',trytime=1):
app_url=GetAppUrl()
token=GetToken(user=self.user)
@ -427,12 +435,12 @@ class GetItemThread(Thread):
for value in values:
item={}
if value.get('folder'):
folder=mongo.db.items.find_one({'id':value['id']})
folder=mon_db.items.find_one({'id':value['id']})
if folder is not None:
if folder['size_order']==value['size']: #文件夹大小未变化，不更新
print(u'path:{},origin size:{},current size:{}--------no change'.format(value['name'],folder['size_order'],value['size']))
else:
mongo.db.items.delete_one({'id':value['id']})
mon_db.items.delete_one({'id':value['id']})
item['type']='folder'
item['user']=self.user
item['order']=0
@ -454,7 +462,7 @@ class GetItemThread(Thread):
path=convert2unicode(value['name'])
path=urllib.unquote('{}:/{}'.format(self.user,path))
item['path']=path
subfodler=mongo.db.items.insert_one(item)
subfodler=mon_db.items.insert_one(item)
if value.get('folder').get('childCount')==0:
continue
else:
@ -463,7 +471,7 @@ class GetItemThread(Thread):
url=app_url+'v1.0/me/drive/root:{}:/children?expand=thumbnails'.format(path)
self.queue.put(dict(url=url,grandid=grandid+1,parent=item['id'],trytime=1))
else:
mongo.db.items.delete_one({'id':value['id']})
mon_db.items.delete_one({'id':value['id']})
item['type']='folder'
item['user']=self.user
item['order']=0
@ -485,7 +493,7 @@ class GetItemThread(Thread):
path=convert2unicode(value['name'])
path=urllib.unquote('{}:/{}'.format(self.user,path))
item['path']=path
subfodler=mongo.db.items.insert_one(item)
subfodler=mon_db.items.insert_one(item)
if value.get('folder').get('childCount')==0:
continue
else:
@ -494,7 +502,7 @@ class GetItemThread(Thread):
url=app_url+'v1.0/me/drive/root:{}:/children?expand=thumbnails'.format(path)
self.queue.put(dict(url=url,grandid=grandid+1,parent=item['id'],trytime=1))
else:
if mongo.db.items.find_one({'id':value['id']}) is not None: #文件存在
if mon_db.items.find_one({'id':value['id']}) is not None: #文件存在
continue
else:
item['type']=GetExt(value['name'])
@ -527,7 +535,7 @@ class GetItemThread(Thread):
item['order']=1
else:
item['order']=2
mongo.db.items.insert_one(item)
mon_db.items.insert_one(item)
else:
print('{}\'s size is zero'.format(url))
if data.get('@odata.nextLink'):
@ -564,7 +572,7 @@ class GetItemThread(Thread):
return data
def clearRedis():
key_lists=['path:*','name:*','*has_item*','*root*']
key_lists=['path:*','name:*','*has_item*','*root*','*:content']
for k in key_lists:
try:
redis_client.delete(*redis_client.keys(k))

74
app/utils/offdownload.py
Unescape View File

@ -7,14 +7,14 @@ def download_and_upload(url,remote_dir,user,gid=None):
down_path=os.path.join(config_dir,'upload')
#重新下载
if gid is not None:
task=mongo.db.down_db.find_one({'gid':gid})
task=mon_db.down_db.find_one({'gid':gid})
if task is None:
return False
if task['up_status']!='100.0%':
new_value={}
new_value['up_status']=u'待机'
new_value['status']=1
mongo.db.down_db.update_many({'gid':gid},{'$set':new_value})
mon_db.down_db.update_many({'gid':gid},{'$set':new_value})
else:
if not url.lower().startswith('http') and not url.lower().startswith('magnet'):
item={}
@ -32,9 +32,9 @@ def download_and_upload(url,remote_dir,user,gid=None):
item['down_status']='-'
item['up_status']='-'
item['status']=-1
mongo.db.down_db.insert_one(item)
mon_db.down_db.insert_one(item)
return
cur_order=mongo.db.down_db.count()
cur_order=mon_db.down_db.count()
option={"dir":down_path,"split":"16","max-connection-per-server":"8","seed-ratio":"0.1","bt-save-metadata":"false","bt-max-peers":"200","header":["User-Agent:Transmission/2.77"]}
item={}
r=p.addUri(url,option)
@ -61,7 +61,7 @@ def download_and_upload(url,remote_dir,user,gid=None):
item['down_status']=u'{}%'.format(round(float(a['completedLength'])/(float(a['totalLength'])+0.1)*100,0))
item['up_status']=u'待机'
item['status']=1
mongo.db.down_db.insert_one(item)
mon_db.down_db.insert_one(item)
while 1:
a=json.loads(p.tellStatus(gid))[0]["result"]
if a.get('followedBy'):
@ -69,8 +69,8 @@ def download_and_upload(url,remote_dir,user,gid=None):
old_status['status']=0
old_status['down_status']='100.0%'
old_status['up_status']=u'磁力文件，无需上传'
mongo.db.down_db.find_one_and_update({'gid':gid},{'$set':old_status})
magnet=re.findall('magnet:\?xt=urn:btih:(.{,40})',mongo.db.down_db.find_one({'gid':gid})['downloadUrl'])[0].lower()+'.torrent'
mon_db.down_db.find_one_and_update({'gid':gid},{'$set':old_status})
magnet=re.findall('magnet:\?xt=urn:btih:(.{,40})',mon_db.down_db.find_one({'gid':gid})['downloadUrl'])[0].lower()+'.torrent'
old_path=os.path.join(down_path,magnet)
try:
os.remove(old_path)
@ -95,16 +95,16 @@ def download_and_upload(url,remote_dir,user,gid=None):
new_item['down_status']=u'{}%'.format(round(float(file['completedLength'])/(float(file['length'])+0.1)*100,0))
new_item['up_status']=u'待机'
new_item['status']=1
mongo.db.down_db.insert_one(new_item)
mon_db.down_db.insert_one(new_item)
a=json.loads(p.tellStatus(gid))[0]["result"]
total=len(a['files'])
complete=0
for idx,file in enumerate(a['files']):
t=mongo.db.down_db.find_one({'gid':gid,'idx':idx})
t=mon_db.down_db.find_one({'gid':gid,'idx':idx})
if t['down_status']=='100.0%':
if t['up_status']=='待机':
new_value['up_status']=u'准备上传'
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
upload_status(gid,idx,remote_dir,user)
elif t['up_status']=='上传成功！':
complete+=1
@ -121,20 +121,20 @@ def download_and_upload(url,remote_dir,user,gid=None):
new_value['localpath']=file['path']
if a['status']=='complete' or (file['completedLength']==file['length'] and int(file['length'])!=0):
new_value['up_status']=u'准备上传'
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
upload_status(gid,idx,remote_dir,user)
complete+=1
elif a['status']=='active' or a['status']=='waiting':
time.sleep(1)
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
elif a['status']=='paused':
new_value['down_status']=u'暂停下载'
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
else:
print('下载出错')
new_value['down_status']=u'下载出错'
new_value['status']=-1
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
complete+=1
# time.sleep(2)
if complete==total:
@ -142,16 +142,16 @@ def download_and_upload(url,remote_dir,user,gid=None):
break
def upload_status(gid,idx,remote_dir,user):
item=mongo.db.down_db.find_one({'gid':gid,'idx':idx})
item=mon_db.down_db.find_one({'gid':gid,'idx':idx})
localpath=item['localpath']
if not remote_dir.endswith('/'):
remote_dir=remote_dir+'/'
remote_path=os.path.join(remote_dir,item['name'])
if not os.path.exists(localpath) and mongo.db.down_db.find_one({'_id':item['_id']})['status']!=0:
if not os.path.exists(localpath) and mon_db.down_db.find_one({'_id':item['_id']})['status']!=0:
new_value={}
new_value['up_status']=u'本地文件不存在。检查：{}'.format(localpath)
new_value['status']=-1
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
return
_upload_session=Upload_for_server(localpath,remote_path,user)
while 1:
@ -180,26 +180,26 @@ def upload_status(gid,idx,remote_dir,user):
elif 'partition upload fail' in msg:
new_value['up_status']='上传失败，已经超过重试次数'
new_value['status']=-1
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
break
elif 'file exists' in msg:
new_value['up_status']='远程文件已存在'
new_value['status']=-1
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
break
elif 'create upload session fail' in msg:
new_value['up_status']='创建实例失败！'
new_value['status']=-1
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
break
else:
new_value['up_status']='上传成功！'
new_value['status']=0
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
time.sleep(2)
os.remove(localpath)
break
mongo.db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
mon_db.down_db.find_one_and_update({'_id':item['_id']},{'$set':new_value})
except Exception as e:
print(e)
break
@ -207,7 +207,7 @@ def upload_status(gid,idx,remote_dir,user):
def get_tasks(status):
tasks=mongo.db.down_db.find({'status':status})
tasks=mon_db.down_db.find({'status':status})
#获取所有的gid
gids=[]
for t in tasks:
@ -223,7 +223,7 @@ def get_tasks(status):
info['files']=[]
total_size=0
complete=0
for file in mongo.db.down_db.find({'gid':gid}):
for file in mon_db.down_db.find({'gid':gid}):
file_info={}
total_size+=int(file['size'])
try:
@ -284,7 +284,7 @@ def Aria2Method(action,**kwargs):
selected_dict.setdefault(gid,[]).append(idx+1)
#之前本就选择下载的gid&idx放进字典
for gid in selected_dict.keys():
tasks=mongo.db.down_db.find({'gid':gid,'selected':'true'})
tasks=mon_db.down_db.find({'gid':gid,'selected':'true'})
for t in tasks:
selected_dict[gid].append(t['idx']+1)
#重新处理可下载文件
@ -301,7 +301,7 @@ def Aria2Method(action,**kwargs):
result.append(info)
for idx in idxs:
new_value={'selected':'true','down_status':'选择下载','status':1}
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':int(idx)-1},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':int(idx)-1},{'$set':new_value})
retdata['result']=result
return retdata
elif action=='unselected':
@ -311,7 +311,7 @@ def Aria2Method(action,**kwargs):
#先创建围表
for gid in kwargs['gids']:
gid,idx=gid.split('#')
nums=mongo.db.down_db.find({'gid':gid}).count()
nums=mon_db.down_db.find({'gid':gid}).count()
if nums<=1:
result=[{'gid':gid,'msg':'当前磁力只有一个文件，无法选择'}]
retdata['result']=result
@ -323,11 +323,11 @@ def Aria2Method(action,**kwargs):
if r=='active':
p.forcePause(gid)
new_value={'selected':'false','down_status':'选择不下载','status':2}
mongo.db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
mon_db.down_db.find_one_and_update({'gid':gid,'idx':idx},{'$set':new_value})
selected_dict.setdefault(gid,[])
#之前本就选择下载的gid&idx放进字典
for gid in selected_dict.keys():
tasks=mongo.db.down_db.find({'gid':gid,'selected':'true'})
tasks=mon_db.down_db.find({'gid':gid,'selected':'true'})
for t in tasks:
selected_dict[gid].append(int(t['idx']+1))
#选择不下载的gid&idx从字典移除
@ -359,7 +359,7 @@ def DBMethod(action,**kwargs):
result=[]
for gid in kwargs['gids']:
info={'gid':gid}
task=mongo.db.down_db.find_one({'gid':gid})
task=mon_db.down_db.find_one({'gid':gid})
if task['down_status']=='100.0%':
info['msg']='文件下载完成！无法更改上传状态'
elif task['down_status']=='下载出错':
@ -370,7 +370,7 @@ def DBMethod(action,**kwargs):
new_value['down_status']='暂停下载'
else:
new_value['down_status']='开始下载'
mongo.db.down_db.update_many({'gid':gid},{'$set':new_value})
mon_db.down_db.update_many({'gid':gid},{'$set':new_value})
info['msg']='更改状态成功'
result.append(info)
elif action in ['remove']:
@ -378,11 +378,11 @@ def DBMethod(action,**kwargs):
for gid in kwargs['gids']:
gid,idx=gid.split('#')
info={'gid':gid,'idx':int(idx)}
task=mongo.db.down_db.find_one({'gid':gid,'idx':int(idx)})
task=mon_db.down_db.find_one({'gid':gid,'idx':int(idx)})
if task['down_status']=='100.0%' and 'partition upload success' in task['up_status']:
info['msg']='正在上传的任务，无法更改状态'
else:
mongo.db.down_db.remove(info)
mon_db.down_db.remove(info)
info['msg']='删除任务成功'
try:
os.remove(task['localpath'])
@ -394,11 +394,11 @@ def DBMethod(action,**kwargs):
result=[]
for gid in kwargs['gids']:
info={'gid':gid}
task=mongo.db.down_db.find_one({'gid':gid})
task=mon_db.down_db.find_one({'gid':gid})
if task['down_status']=='100.0%' and 'partition upload success' in task['up_status']:
info['msg']='正在上传的任务，无法更改状态'
else:
mongo.db.down_db.delete_many(info)
mon_db.down_db.delete_many(info)
info['msg']='删除任务成功'
try:
os.delete_many(task['localpath'])
@ -411,9 +411,9 @@ def DBMethod(action,**kwargs):
for gid in kwargs['gids']:
info={'gid':gid}
new_value={'status':1}
mongo.db.down_db.update_many({'gid':gid},{'$set':new_value})
mon_db.down_db.update_many({'gid':gid},{'$set':new_value})
info['msg']='更改状态成功'
it=mongo.db.down_db.find_one({'gid':gid})
it=mon_db.down_db.find_one({'gid':gid})
user=it['user']
remote_dir=it['remote_dir']
cmd=u'python {} download_and_upload "{}" "{}" {} {}'.format(os.path.join(config_dir,'function.py'),1,remote_dir,user,gid)

106
app/utils/updatefile.py
Unescape View File

@ -1,72 +1,13 @@
#-*- coding=utf-8 -*-
from header import *
def Dir(path=u'A:/'):
app_url=GetAppUrl()
user,n_path=path.split(':')
print('update {}\'s file'.format(user))
if n_path=='/':
BaseUrl=app_url+u'v1.0/me/drive/root/children?expand=thumbnails'
# mongo.db.items.remove()
queue=Queue()
# queue.put(dict(url=BaseUrl,grandid=grandid,parent=parent,trytime=1))
g=GetItemThread(queue,user)
g.GetItem(BaseUrl)
queue=g.queue
else:
grandid=0
parent=''
if n_path.endswith('/'):
n_path=n_path[:-1]
if not n_path.startswith('/'):
n_path='/'+n_path
n_path=urllib.quote(n_path.encode('utf-8'))
BaseUrl=app_url+u'v1.0/me/drive/root:{}:/children?expand=thumbnails'.format(n_path)
queue=Queue()
# queue.put(dict(url=BaseUrl,grandid=grandid,parent=parent,trytime=1))
g=GetItemThread(queue,user)
g.GetItem(BaseUrl,grandid,parent,1)
queue=g.queue
if queue.qsize()==0:
return
tasks=[]
for i in range(min(10,queue.qsize())):
t=GetItemThread(queue,user)
# t.setDaemon(True)
t.start()
tasks.append(t)
# for t in tasks:
# t.join()
error_status=0
while 1:
for t in tasks:
print('thread {}\'s status {},qsize {}'.format(t.getName(),t.isAlive(),t.queue.qsize()))
if t.isAlive()==False and t.queue.qsize()==0:
tasks.pop(tasks.index(t))
t.stop()
if t.queue.qsize()==0 and t.isAlive()==True:
error_status+=1
print('error status times:{}'.format(error_status))
else:
error_status=1
if error_status>=20 and t in tasks:
print('force kill thread:{}'.format(t.getName()))
tasks.pop(tasks.index(t))
t.stop()
if len(tasks)==0:
print(u'{} all thread stop!'.format(path))
break
time.sleep(1)
RemoveRepeatFile()
def Dir_all(path=u'A:/'):
app_url=GetAppUrl()
user,n_path=path.split(':')
print('update {}\'s {} file'.format(user,n_path))
if n_path=='/':
BaseUrl=app_url+u'v1.0/me/drive/root/children?expand=thumbnails'
mongo.db.items.remove({'user':user})
mon_db.items.remove({'user':user})
queue=Queue()
g=GetItemThread(queue,user)
g.GetItem(BaseUrl)
@ -78,14 +19,14 @@ def Dir_all(path=u'A:/'):
n_path=n_path[:-1]
if not n_path.startswith('/'):
n_path='/'+n_path
if mongo.db.items.find_one({'grandid':0,'type':'folder','user':user}):
if mon_db.items.find_one({'grandid':0,'type':'folder','user':user}):
parent_id=0
for idx,p in enumerate(n_path[1:].split('/')):
if parent_id==0:
parent_id=mongo.db.items.find_one({'name':p,'grandid':idx,'user':user})['id']
parent_id=mon_db.items.find_one({'name':p,'grandid':idx,'user':user})['id']
else:
parent_id=mongo.db.items.find_one({'name':p,'grandid':idx,'parent':parent_id})['id']
mongo.db.items.delete_many({'parent':parent_id})
parent_id=mon_db.items.find_one({'name':p,'grandid':idx,'parent':parent_id})['id']
mon_db.items.delete_many({'parent':parent_id})
grandid=idx+1
parent=parent_id
n_path=urllib.quote(n_path.encode('utf-8'))
@ -110,7 +51,6 @@ def Dir_all(path=u'A:/'):
print('thread {}\'s status {},qsize {}'.format(t.getName(),t.isAlive(),t.queue.qsize()))
if t.isAlive()==False and t.queue.qsize()==0:
tasks.pop(tasks.index(t))
t.stop()
if t.queue.qsize()==0 and t.isAlive()==True:
error_status+=1
print('error status times:{}'.format(error_status))
@ -119,38 +59,23 @@ def Dir_all(path=u'A:/'):
if error_status>=20 and t in tasks:
print('force kill thread:{}'.format(t.getName()))
tasks.pop(tasks.index(t))
t.stop()
if len(tasks)==0:
print(u'{} all thread stop!'.format(path))
break
time.sleep(1)
RemoveRepeatFile()
# RemoveRepeatFile()
def GetRootid(user='A'):
key='{}:rootid'.format(user)
if redis_client.exists(key):
return redis_client.get(key)
else:
app_url=GetAppUrl()
token=GetToken(user=user)
url=app_url+u'v1.0/me/drive/root/'
headers={'Authorization': 'Bearer {}'.format(token)}
headers.update(default_headers)
r=requests.get(url,headers=headers)
data=json.loads(r.content)
redis_client.set(key,data['id'],3600)
return data['id']
def UpdateFile(renew='all'):
tasks=[]
if renew=='all':
mongo.db.items.remove()
mon_db.items.delete_many({})
clearRedis()
for user,item in od_users.items():
if item.get('client_id')!='':
share_path='{}:{}'.format(user,item['share_path'])
# Dir_all(share_path)
t=Thread(target=Dir_all,args=(share_path,))
t=Thread(target=Dir,args=(share_path,))
t.start()
tasks.append(t)
for t in tasks:
@ -175,6 +100,21 @@ def UpdateFile(renew='all'):
os.kill(os.getpid(), signal.SIGKILL)
def GetRootid(user='A'):
key='{}:rootid'.format(user)
if redis_client.exists(key):
return redis_client.get(key)
else:
app_url=GetAppUrl()
token=GetToken(user=user)
url=app_url+u'v1.0/me/drive/root/'
headers={'Authorization': 'Bearer {}'.format(token)}
headers.update(default_headers)
r=requests.get(url,headers=headers)
data=json.loads(r.content)
redis_client.set(key,data['id'],3600)
return data['id']
def FileExists(filename,user='A'):
token=GetToken(user=user)

2
app/utils/upload.py
Unescape View File

@ -242,7 +242,7 @@ class MultiUpload(Thread):
while not self.queue.empty():
localpath,remote_dir=self.queue.get()
cp='{}:/{}'.format(self.user,remote_dir)
if mongo.db.items.find_one({'path':cp}):
if mon_db.items.find_one({'path':cp}):
print(u'{} exists!'.format(cp))
else:
Upload(localpath,remote_dir,self.user)

2
config.py
Unescape View File

@ -7,8 +7,6 @@ basedir = os.path.abspath(os.path.dirname(__file__))
class config:
SECRET_KEY = os.path.join(config_dir,'PyOne'+password)
CACHE_TYPE='redis'
MONGO_URI="mongodb://127.0.0.1:27017/three"
REDIS2_URL='redis://127.0.01:6379/0'
@staticmethod
def init_app(app):

Write Preview
Loading…
Cancel
Save