1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
| from qiniu import Auth, put_file, etag, urlsafe_base64_encode, BucketManager, CdnManager
from typing import List, Dict
import os
from qiniu import build_batch_delete
class Sync:
"""
同步目录至七牛云
"""
def __init__(
self,
access_key: str,
secret_key: str,
bucket_name: str,
sync_dir: str,
exclude: List,
cover: bool,
remove_redundant: bool,
host: str,
):
self.bucket_name = bucket_name
self.q = Auth(access_key, secret_key)
self.bucket = BucketManager(self.q)
self.sync_dir = sync_dir
self.exclude = exclude
self.cover = cover
self.remove_redundant = remove_redundant
self.host = host
self.sync()
def sync(self):
"""
同步操作
:return:
"""
remote_files = self.list_remote()
local_files = self.list_local()
# 首先删除远端仓库中多余的文件
remove_remote_files = []
for remote_filename in remote_files:
if remote_filename not in local_files:
remove_remote_files.append(remote_filename)
print('delete remote file size = ' + str(len(remove_remote_files)))
self.bucket.batch(build_batch_delete(self.bucket_name, remove_remote_files))
cdnManager = CdnManager(self.q)
refreshUrls = []
refreshDirs = [self.host]
# 上传本地文件到远端(仅上传远端不存在的以及修改过的)
for local_filename in local_files:
if (
local_filename not in remote_files
or local_files[local_filename]["hash"]
!= remote_files[local_filename]["hash"]
):
ret, info = put_file(
self.q.upload_token(self.bucket_name, local_filename, 3600),
local_filename,
local_files[local_filename]["fullpath"],
)
remotepath = self.host + ret["key"]
print('uploaded: ' + remotepath)
#refreshUrls.append(remotepath)
print('uploaded size = ' + len(refreshUrls))
# 刷新节点资源
cdnManager.refresh_dirs(refreshDirs)
print('sycn completed !!!!')
def list_remote(self) -> Dict:
"""
列出远程仓库所有的文件信息
:return: List
"""
result = {}
marker = None
ret = self.bucket.list(self.bucket_name, marker=marker, limit=500)[0]
for file in ret["items"]:
result[file["key"]] = file
if("marker" in ret):
marker = ret["marker"]
while(marker):
print('marker = ' + marker)
ret = self.bucket.list(self.bucket_name, marker=marker, limit=500)[0]
for file in ret["items"]:
result[file["key"]] = file
if("marker" in ret):
marker = ret["marker"]
else:
break
print('pull remote file info success')
return result
def list_local(self) -> Dict:
"""
列出本地仓库所有的文件信息
"""
files = {}
def get_files(path):
for filename in os.listdir(path):
if filename in self.exclude:
continue
fullpath = os.path.join(path, filename)
if os.path.isfile(fullpath):
key = fullpath.split(self.sync_dir)[1]
foldpath = key.replace("\\","/") #实际是相对路径
files[foldpath] = {"fullpath": fullpath, "hash": etag(fullpath)}
else:
get_files(fullpath)
get_files(self.sync_dir)
print('list local file info success')
return files
if __name__ == "__main__":
Sync(
access_key="nzt90pr6nkba0dG1TD2oMul0XHBe6ALuBo3hHc_l", # access_key
secret_key="yoRI0b8J9dDiYwtJeE844tYbrupdjSiVio6sczf1", # secret_key
bucket_name="blog", # bucket_name
#sync_dir="~/blog/public/", # 静态文件目录(后面必须有斜杠/)
sync_dir="D:\\blog\\gitlab\\public\\",
exclude=[".DS_Store"],
cover=True,
remove_redundant=True,
host="https://dp2px.com/",
)
|
七牛对象存储maxAge设置
GitLab CI/CD
