转载

基于七牛Python SDK写的一个同步脚本

需求背景

最近刚搭了个 markdown静态博客，想把博客的图片放到云存储中。

经过调研觉得七牛可以满足我个人的需求，就选它了。

博客要引用图片就要先将图片上传到云上。

虽然七牛网站后台可以上传文件，但每次上传都需要先登录，然后选择图片，设置连接地址，才能上传。

这个过程有些繁琐，所以我便想用七牛云提供的SDK写个一同步工具，方便增量同步文件。

有了这个想法，就马上行动。花了大概一个上午的时间，总算把这个工具给写出来，

并放到 GitOSC 和 github 上。

#!/usr/bin/env python #-*- coding:utf-8 -*- #  # AUTHOR = "heqingpan" # AUTHOR_EMAIL = "heqingpan@126.com" # URL = "http://git.oschina.net/hqp/qiniu_sync" import qiniu from qiniu import Auth from qiniu import BucketManager import os import re access_key = '' secret_key = '' bucket_name = '' bucket_domain = '' q = Auth(access_key, secret_key) bucket = BucketManager(q) basedir=os.path.realpath(os.path.dirname(__file__)) filename=__file__ ignore_paths=[filename,"{0}c".format(filename)] ignore_names=[".DS_Store",".git",".gitignore"] charset="utf8" diff_time=2*60 def list_all(bucket_name, bucket=None, prefix="", limit=100):  rlist=[]  if bucket is None:   bucket = BucketManager(q)  marker = None  eof = False  while eof is False:   ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit)   marker = ret.get('marker', None)   for item in ret['items']:    rlist.append(item["key"])  if eof is not True:   # 错误处理   #print "error"   pass  return rlist def get_files(basedir="",fix="",rlist=None,ignore_paths=[],ignore_names=[]):  if rlist is None:   rlist=[]  for subfile in os.listdir(basedir):   temp_path=os.path.join(basedir,subfile)   tp=os.path.join(fix,subfile)   if tp in ignore_names:    continue   if tp in ignore_paths:    continue   if os.path.isfile(temp_path):    rlist.append(tp)   elif os.path.isdir(temp_path):    get_files(temp_path,tp,rlist,ignore_paths,ignore_names)  return rlist def get_valid_key_files(subdir=""):  basedir=subdir or basedir  files = get_files(basedir=basedir,ignore_paths=ignore_paths,ignore_names=ignore_names)  return map(lambda f:(f.replace("//","/"),f),files) def sync():  qn_keys=list_all(bucket_name,bucket)  qn_set=set(qn_keys)  l_key_files=get_valid_key_files(basedir)  k2f={}  update_keys=[]  u_count=500  u_index=0  for k,f in l_key_files:   k2f[k]=f   str_k=k   if isinstance(k,str):    k=k.decode(charset)   if k in qn_set:    update_keys.append(str_k)    u_index+=1    if u_index > u_count:     u_index-=u_count     update_file(k2f,update_keys)     update_keys=[]   else:    # upload    upload_file(k,os.path.join(basedir,f))  if update_keys:   update_file(k2f,update_keys)  print "sync end" def update_file(k2f,ulist):  ops=qiniu.build_batch_stat(bucket_name,ulist)  rets,infos = bucket.batch(ops)  for i in xrange(len(ulist)):   k=ulist[i]   f=k2f.get(k)   ret=rets[i]["data"]   size=ret.get("fsize",None)   put_time = int(ret.get("putTime")/10000000)   local_size=os.path.getsize(f)   local_time=int(os.path.getatime(f))   if local_size==size:    continue   if put_time >= local_time - diff_time:    # is new    continue   # update   upload_file(k,os.path.join(basedir,f)) def upload_file(key,localfile):  print "upload_file:"  print key  token = q.upload_token(bucket_name, key)  mime_type = get_mime_type(localfile)  params = {'x:a': 'a'}  progress_handler = lambda progress, total: progress  ret, info = qiniu.put_file(token, key, localfile, params, mime_type, progress_handler=progress_handler) def get_mime_type(path):  mime_type = "text/plain"  return mime_type def main():  sync() if __name__=="__main__":  main()

这个同步脚本支持批量比较文件，差异增量更新、批量更新。

使用方式

安装七牛Python SDK
pip install qiniu
填写脚本文件(qiniusync.py)的配置信息
```
access_key = '' secret_key = '' bucket_name = ''
```
注册后可以拿到对应的信息
将脚本文件(qiniusync.py)拷贝到待同步根目录
运行脚本
```
python qiniusync.py
```