update uploadId cache

Author: carsonxu

dist/cos-js-sdk-v5.js

@@ -4,8 +4,8 @@
   "description": "JavaScript SDK for [腾讯云对象存储](https://cloud.tencent.com/product/cos)",
   "main": "index.js",
   "scripts": {
-    "dev": "cross-env NODE_ENV=development webpack -w",
     "server": "node server/sts.js",
+    "dev": "cross-env NODE_ENV=development webpack -w",
     "build": "cross-env NODE_ENV=production webpack",
     "cos-auth.min.js": "uglifyjs ./demo/common/cos-auth.js -o ./demo/common/cos-auth.min.js -c -m"
   },

dist/cos-js-sdk-v5.min.js

@@ -1,3 +1,4 @@
+var session = require('./session');
 var Async = require('./async');
 var EventProxy = require('./event').EventProxy;
 var util = require('./util');
@@ -41,13 +42,13 @@ function sliceUploadFile(params, callback) {
             SliceList: UploadData.SliceList,
         }, function (err, data) {
             if (!self._isRunningTask(TaskId)) return;
-            delete uploadIdUsing[UploadData.UploadId];
+            session.removeUsing(UploadData.UploadId);
             if (err) {
                 onProgress(null, true);
                 return ep.emit('error', err);
             }
+            session.removeUploadId(UploadData.UploadId);
             onProgress({loaded: FileSize, total: FileSize}, true);
-            removeUploadId.call(self, UploadData.UploadId);
             ep.emit('upload_complete', data);
         });
     });
@@ -56,14 +57,9 @@ function sliceUploadFile(params, callback) {
     ep.on('get_upload_data_finish', function (UploadData) {
 
         // 处理 UploadId 缓存
-        var uuid = util.getFileUUID(Body, params.ChunkSize);
-        uuid && setUploadId.call(self, uuid, UploadData.UploadId); // 缓存 UploadId
-        uploadIdUsing[UploadData.UploadId] = true; // 标记 UploadId 为正在使用
-        TaskId && self.on('inner-kill-task', function (data) {
-            if (data.TaskId === TaskId && data.toState === 'canceled') {
-                delete uploadIdUsing[UploadData.UploadId]; // 去除 UploadId 正在使用的标记
-            }
-        });
+        var uuid = session.getFileId(Body, params.ChunkSize, Bucket, Key);
+        uuid && session.saveUploadId(uuid, UploadData.UploadId, self.options.UploadIdCacheLimit); // 缓存 UploadId
+        session.setUsing(UploadData.UploadId); // 标记 UploadId 为正在使用
 
         // 获取 UploadId
         onProgress(null, true); // 任务状态开始 uploading
@@ -157,75 +153,6 @@ function sliceUploadFile(params, callback) {
 
 }
 
-
-// 按照文件特征值，缓存 UploadId
-var uploadIdCache;
-var uploadIdUsing = {};
-var uploadIdCacheKey = 'cos_sdk_upload_cache';
-function initUploadId() {
-    var cacheLimit = this.options.UploadIdCacheLimit;
-    if (!uploadIdCache) {
-        if (cacheLimit) {
-            try {
-                uploadIdCache = JSON.parse(localStorage.getItem(uploadIdCacheKey)) || [];
-            } catch (e) {}
-        }
-        if (!uploadIdCache) {
-            uploadIdCache = [];
-        }
-    }
-}
-function setUploadId(uuid, UploadId, isDisabled) {
-    initUploadId.call(this);
-    for (var i = uploadIdCache.length - 1; i >= 0; i--) {
-        if (uploadIdCache[i][0] === uuid && uploadIdCache[i][1] === UploadId) {
-            uploadIdCache.splice(i, 1);
-        }
-    }
-    uploadIdCache.unshift([uuid, UploadId]);
-    var cacheLimit = this.options.UploadIdCacheLimit;
-    if (uploadIdCache.length > cacheLimit) {
-        uploadIdCache.splice(cacheLimit);
-    }
-    cacheLimit && setTimeout(function () {
-        try {
-            localStorage.setItem(uploadIdCacheKey, JSON.stringify(uploadIdCache));
-        } catch (e) {}
-    });
-}
-function removeUploadId(UploadId) {
-    initUploadId.call(this);
-    delete uploadIdUsing[UploadId];
-    for (var i = uploadIdCache.length - 1; i >= 0; i--) {
-        if (uploadIdCache[i][1] === UploadId) {
-            uploadIdCache.splice(i, 1)
-        }
-    }
-    var cacheLimit = this.options.UploadIdCacheLimit;
-    if (uploadIdCache.length > cacheLimit) {
-        uploadIdCache.splice(cacheLimit);
-    }
-    cacheLimit && setTimeout(function () {
-        try {
-            if (uploadIdCache.length) {
-                localStorage.setItem(uploadIdCacheKey, JSON.stringify(uploadIdCache));
-            } else {
-                localStorage.removeItem(uploadIdCacheKey);
-            }
-        } catch (e) {}
-    });
-}
-function getUploadId(uuid) {
-    initUploadId.call(this);
-    var CacheUploadIdList = [];
-    for (var i = 0; i < uploadIdCache.length; i++) {
-        if (uploadIdCache[i][0] === uuid) {
-            CacheUploadIdList.push(uploadIdCache[i][1]);
-        }
-    }
-    return CacheUploadIdList.length ? CacheUploadIdList : null;
-}
-
 // 获取上传任务的 UploadId
 function getUploadIdAndPartList(params, callback) {
     var TaskId = params.TaskId;
@@ -316,7 +243,7 @@ function getUploadIdAndPartList(params, callback) {
     });
 
     // 存在 UploadId
-    ep.on('upload_id_ready', function (UploadData) {
+    ep.on('upload_id_available', function (UploadData) {
         // 转换成 map
         var map = {};
         var list = [];
@@ -364,18 +291,18 @@ function getUploadIdAndPartList(params, callback) {
             if (!UploadId) {
                 return callback({Message: 'no upload id'});
             }
-            ep.emit('upload_id_ready', {UploadId: UploadId, PartList: []});
+            ep.emit('upload_id_available', {UploadId: UploadId, PartList: []});
         });
     });
 
     // 如果已存在 UploadId，找一个可以用的 UploadId
-    ep.on('has_upload_id', function (UploadIdList) {
+    ep.on('has_and_check_upload_id', function (UploadIdList) {
         // 串行地，找一个内容一致的 UploadId
         UploadIdList = UploadIdList.reverse();
         Async.eachLimit(UploadIdList, 1, function (UploadId, asyncCallback) {
             if (!self._isRunningTask(TaskId)) return;
             // 如果正在上传，跳过
-            if (uploadIdUsing[UploadId]) {
+            if (session.using[UploadId]) {
                 asyncCallback(); // 检查下一个 UploadId
                 return;
             }
@@ -388,7 +315,7 @@ function getUploadIdAndPartList(params, callback) {
             }, function (err, PartListData) {
                 if (!self._isRunningTask(TaskId)) return;
                 if (err) {
-                    removeUploadId.call(self, UploadId);
+                    session.removeUsing(UploadId);
                     return ep.emit('error', err);
                 }
                 var PartList = PartListData.PartList;
@@ -414,7 +341,7 @@ function getUploadIdAndPartList(params, callback) {
             if (!self._isRunningTask(TaskId)) return;
             onHashProgress(null, true);
             if (AvailableUploadData && AvailableUploadData.UploadId) {
-                ep.emit('upload_id_ready', AvailableUploadData);
+                ep.emit('upload_id_available', AvailableUploadData);
             } else {
                 ep.emit('no_available_upload_id');
             }
@@ -424,50 +351,52 @@ function getUploadIdAndPartList(params, callback) {
     // 在本地缓存找可用的 UploadId
     ep.on('seek_local_avail_upload_id', function (RemoteUploadIdList) {
         // 在本地找可用的 UploadId
-        var uuid = util.getFileUUID(params.Body, params.ChunkSize), LocalUploadIdList;
-        if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
-            var next = function (index) {
-                // 如果本地找不到可用 UploadId，再一个个遍历校验远端
-                if (index >= LocalUploadIdList.length) {
-                    ep.emit('has_upload_id', RemoteUploadIdList);
-                    return;
-                }
-                var UploadId = LocalUploadIdList[index];
-                // 如果不在远端 UploadId 列表里，跳过并删除
-                if (!util.isInArray(RemoteUploadIdList, UploadId)) {
-                    removeUploadId.call(self, UploadId);
-                    next(index + 1);
-                    return;
-                }
-                // 如果正在上传，跳过
-                if (uploadIdUsing[UploadId]) {
+        var uuid = session.getFileId(params.Body, params.ChunkSize, Bucket, Key);
+        var LocalUploadIdList = session.getUploadIdList(uuid);
+        if (!uuid || !LocalUploadIdList) {
+            ep.emit('has_and_check_upload_id', RemoteUploadIdList);
+            return;
+        }
+        var next = function (index) {
+            // 如果本地找不到可用 UploadId，再一个个遍历校验远端
+            if (index >= LocalUploadIdList.length) {
+                ep.emit('has_and_check_upload_id', RemoteUploadIdList);
+                return;
+            }
+            var UploadId = LocalUploadIdList[index];
+            // 如果不在远端 UploadId 列表里，跳过并删除
+            if (!util.isInArray(RemoteUploadIdList, UploadId)) {
+                session.removeUploadId(UploadId);
+                next(index + 1);
+                return;
+            }
+            // 如果正在上传，跳过
+            if (session.using[UploadId]) {
+                next(index + 1);
+                return;
+            }
+            // 判断 UploadId 是否存在线上
+            wholeMultipartListPart.call(self, {
+                Bucket: Bucket,
+                Region: Region,
+                Key: Key,
+                UploadId: UploadId,
+            }, function (err, PartListData) {
+                if (!self._isRunningTask(TaskId)) return;
+                if (err) {
+                    // 如果 UploadId 获取会出错，跳过并删除
+                    session.removeUploadId(UploadId);
                     next(index + 1);
-                    return;
+                } else {
+                    // 找到可用 UploadId
+                    ep.emit('upload_id_available', {
+                        UploadId: UploadId,
+                        PartList: PartListData.PartList,
+                    });
                 }
-                // 判断 UploadId 是否存在线上
-                wholeMultipartListPart.call(self, {
-                    Bucket: Bucket,
-                    Region: Region,
-                    Key: Key,
-                    UploadId: UploadId,
-                }, function (err, PartListData) {
-                    if (!self._isRunningTask(TaskId)) return;
-                    if (err) {
-                        removeUploadId.call(self, UploadId);
-                        next(index + 1);
-                    } else {
-                        // 找到可用 UploadId
-                        ep.emit('upload_id_ready', {
-                            UploadId: UploadId,
-                            PartList: PartListData.PartList,
-                        });
-                    }
-                });
-            };
-            next(0);
-        } else {
-            ep.emit('has_upload_id', RemoteUploadIdList);
-        }
+            });
+        };
+        next(0);
     });
 
     // 获取线上 UploadId 列表
@@ -491,10 +420,11 @@ function getUploadIdAndPartList(params, callback) {
             if (RemoteUploadIdList.length) {
                 ep.emit('seek_local_avail_upload_id', RemoteUploadIdList);
             } else {
-                var uuid = util.getFileUUID(params.Body, params.ChunkSize), LocalUploadIdList;
-                if (uuid && (LocalUploadIdList = getUploadId.call(self, uuid))) {
+                // 远端没有 UploadId，清理缓存的 UploadId
+                var uuid = session.getFileId(params.Body, params.ChunkSize, Bucket, Key), LocalUploadIdList;
+                if (uuid && (LocalUploadIdList = session.getUploadIdList(uuid))) {
                     util.each(LocalUploadIdList, function (UploadId) {
-                        removeUploadId.call(self, UploadId);
+                        session.removeUploadId(UploadId);
                     });
                 }
                 ep.emit('no_available_upload_id');

package.json

@@ -1611,9 +1611,8 @@ function restoreObject(params, callback) {
 function multipartInit(params, callback) {
 
     var self = this;
-    var headers = params.Headers;
-
     // 特殊处理 Cache-Control
+    var headers = params.Headers;
     !headers['Cache-Control'] && (headers['Cache-Control'] = '');
     util.getBodyMd5(params.Body && (params.UploadAddMetaMd5 || self.options.UploadAddMetaMd5), params.Body, function (md5) {
         if (md5) params.Headers['x-cos-meta-md5'] = md5;

src/advance.js

@@ -0,0 +1,89 @@
+var util = require('./util');
+
+// 按照文件特征值，缓存 UploadId
+var cacheKey = 'cos_sdk_upload_cache';
+var expires = 30 * 24 * 3600;
+var cache;
+var timer;
+
+var init = function () {
+    if (cache) return;
+    cache = JSON.parse(localStorage.getItem(cacheKey) || '[]') || [];
+    // 清理太老旧的数据
+    var changed = false;
+    var now = Math.round(Date.now() / 1000);
+    for (var i = cache.length - 1; i >= 0; i--) {
+        var mtime = cache[i][2];
+        if (!mtime || mtime + expires < now) {
+            cache.splice(i, 1);
+            changed = true;
+        }
+    }
+    changed && localStorage.setItem(cacheKey, JSON.stringify(cache));
+};
+
+// 把缓存存到本地
+var save = function () {
+    if (timer) return;
+    timer = setTimeout(function () {
+        localStorage.setItem(cacheKey, JSON.stringify(cache));
+        timer = null;
+    }, 400);
+};
+
+var mod = {
+    using: {},
+    // 标记 UploadId 正在使用
+    setUsing: function (uuid) {
+        mod.using[uuid] = true;
+    },
+    // 标记 UploadId 已经没在使用
+    removeUsing: function (uuid) {
+        delete mod.using[uuid];
+    },
+    // 用上传参数生成哈希值
+    getFileId: function (file, ChunkSize, Bucket, Key) {
+        if (file.name && file.size && file.lastModifiedDate && ChunkSize) {
+            return util.md5([file.name, file.size, file.lastModifiedDate, ChunkSize, Bucket, Key].join('::'));
+        } else {
+            return null;
+        }
+    },
+    // 获取文件对应的 UploadId 列表
+    getUploadIdList: function (uuid) {
+        if (!uuid) return null;
+        init();
+        var list = [];
+        for (var i = 0; i < cache.length; i++) {
+            if (cache[i][0] === uuid)
+                list.push(cache[i][1]);
+        }
+        return list.length ? list : null;
+    },
+    // 缓存 UploadId
+    saveUploadId: function (uuid, UploadId, limit) {
+        init();
+        if (!uuid) return;
+        // 清理没用的 UploadId
+        for (var i = cache.length - 1; i >= 0; i--) {
+            var item = cache[i];
+            if (item[0] === uuid && item[1] === UploadId) {
+                cache.splice(i, 1);
+            }
+        }
+        cache.unshift([uuid, UploadId, Math.round(Date.now() / 1000)]);
+        if (cache.length > limit) cache.splice(limit);
+        save();
+    },
+    // UploadId 已用完，移除掉
+    removeUploadId: function (UploadId) {
+        init();
+        delete mod.using[UploadId];
+        for (var i = cache.length - 1; i >= 0; i--) {
+            if (cache[i][1] === UploadId) cache.splice(i, 1)
+        }
+        save();
+    },
+};
+
+module.exports = mod;