日常如果是上传一些小文件，在程序实现中，我们都是直接上传，一般都没什么问题。如果 针对大文件上传的业务中，就会面临着：
1、网速问题，导致文件上传超时，而导致失败。
2、效率问题，上传大文件等待时间过长，如果是需要上传多个，就会更慢。
3、体验问题，用户无法预知上传还需花费的时间，系统没有及时反馈，用户无法判断文件是否还在上传，还是断开。
这时候就需要采用分布式文件上传系统。
项目简介
这是一个基于.Net Core 构建的简单、跨平台分布式文件上传系统，支持分块上传、多个项目同时上传、接口权限控制采用JWT机制。
技术架构
1、跨平台：这是基于.Net Core开发的系统，可以部署在Docker, Windows, Linux, Mac。
2、.Net 2.1 + Jwt + simple-uploader
项目结构
项目分为分块上传与一般上传Demo，Web、控制台上传Demo。ufs项目是分布式文件上传的统一接口，ufs会根据配置把上传的文件发到ufs.node节点，ufs.node会把上传成功路径返回给ufs并存储，用户访问的时候，ufs会访问对应节点返回资源。
UploadServer为一般文件上传接口，UploadServer.FrontEndDemo为Web上传文件Demo。
使用
1、配置
配置允许上传域名、服务接口地址、允许的文件格式、文件大小、存储路径等信息。
{"AllowedHosts": "*","urls": "http://localhost:6001","uploadServer": {"rootUrl": "http://localhost:6001","entryPoint1": "/upload","entryPoint2": "/chunkUpload","virtualPath": "","physicalPath": "/Users/loogn/Desktop/uploader","appendMimes": ".htm3:text/html;","responseCache": 604800,"jwtSecret": "1234561234","limitSize": "20mb","allowExts": ".txt;.jpg;.jpeg;.png;.doc;.docx;.xls;.xlsx;.ppt;.pptx;.pdf","apps": {"default": {"allowOrigins": "","enableThumbnail": true,"limitExts": ".exe;","thumbnailExts": ".jpg;.jpeg;.png;"},"app1": {"allowOrigins": "*"}}}}
2、前端
一般上传代码
$("#file1").change(function () {$.ajaxFileUpload({fileElementId: 'file1',url: 'http://localhost:6001/upload',dataType: 'text',//success: function (data) {console.log("上传成功：", data);},data: {"jwt": jwt}});});
分块上传
var uploader = new Uploader({target: 'http://localhost:6001/chunkupload',headers: {jwt: jwt}});uploader.assignBrowse(document.getElementById('browseButton'));//uploader.assignBrowse(document.getElementById('folderButton'), true);//// 文件添加 单个文件uploader.on('fileAdded', function (file, event) {console.log("fileAdded:", file, event)});// 单个文件上传成功uploader.on('fileSuccess', function (rootFile, file, message) {console.log("fileSuccess：", rootFile, file, message)});// 根下的单个文件（文件夹）上传完成uploader.on('fileComplete', function (rootFile) {console.log("fileComplete:", rootFile)});// 某个文件上传失败了uploader.on('fileError', function (rootFile, file, message) {console.log("fileError:", rootFile, file, message)});
3、后端
一般上传
public async Task InvokeAsync(HttpContext context, RequestDelegate next){context.Response.Headers.Add("Access-Control-Allow-Origin", "*");context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)){context.Response.StatusCode = (int) HttpStatusCode.OK;}else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)){//验证jwtstring token = null;if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)){token = jwt.ToString();}else if (context.Request.Form.TryGetValue("jwt", out jwt)){token = jwt.ToString();}else{await context.Response.WriteAsync(new UploadResult(){msg = "No JWT in the header and form"}.toJson());return;}try{var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature().Decode<JwtPayload>(token);var msg = payload.validate();if (msg != null){await context.Response.WriteAsync(new UploadResult(){msg = msg}.toJson());return;}//特定的配置var appConfig = _config.GetAppConfig(payload.app);//跨域context.Request.Headers.TryGetValue("Origin", out var origins);var origin = origins.ToString();if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)){context.Response.Headers.Add("Access-Control-Allow-Origin", origin);}//获取上传的文件var file = context.Request.Form.Files.FirstOrDefault();if (file == null || file.Length == 0){await context.Response.WriteAsync(new UploadResult(){msg = "There is no file data"}.toJson());return;}//大小验证if (file.Length > (payload.GetByteSize() ?? _config.GetByteSize())){await context.Response.WriteAsync(new UploadResult(){msg = "The file is too big"}.toJson());return;}//后缀验证var ext = Path.GetExtension(file.FileName);if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)){await context.Response.WriteAsync(new UploadResult(){msg = "File extension is not allowed"}.toJson());return;}//上传逻辑var now = DateTime.Now;var yy = now.ToString("yyyy");var mm = now.ToString("MM");var dd = now.ToString("dd");var fileName = Guid.NewGuid().ToString("n") + ext;var folder = Path.Combine(_config.PhysicalPath, payload.app, yy, mm, dd);if (!Directory.Exists(folder)){Directory.CreateDirectory(folder);}var filePath = Path.Combine(folder, fileName);using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write)){file.CopyTo(fileStream);fileStream.Flush(true);}var fileUrl = _config.RootUrl + "/" + payload.app + "/" + yy + "/" + mm +"/" +dd +"/" + fileName;await context.Response.WriteAsync(new UploadResult(){ok = true,url = fileUrl}.toJson());}catch (TokenExpiredException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has expired"}.toJson());}catch (SignatureVerificationException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has invalid signature"}.toJson());}}else{await context.Response.WriteAsync(new UploadResult(){msg = $"Request method '{context.Request.Method}' is not supported"}.toJson());}}
分块上传
public async Task InvokeAsync(HttpContext context, RequestDelegate next){context.Response.Headers.Add("Access-Control-Allow-Origin", "*");context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)){context.Response.StatusCode = (int)HttpStatusCode.OK;}else if (context.Request.Method.Equals(HttpMethods.Get, StringComparison.OrdinalIgnoreCase)){//简单实现context.Request.Query.TryGetValue("chunkNumber", out var chunkNumbers);int.TryParse(chunkNumbers.ToString(), out var chunkNumber);context.Request.Query.TryGetValue("identifier", out var identifiers);if (chunkNumber == 0 || string.IsNullOrEmpty(identifiers)){context.Response.StatusCode = 204;}else{var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifiers);if (File.Exists(chunkFilename)){await context.Response.WriteAsync("found");}else{context.Response.StatusCode = 204;}}}else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)){//验证jwtstring token = null;if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)){token = jwt.ToString();}else if (context.Request.Form.TryGetValue("jwt", out jwt)){token = jwt.ToString();}else{await context.Response.WriteAsync(new UploadResult(){msg = "No JWT in the header and form"}.toJson());return;}try{var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature().Decode<JwtPayload>(token);var msg = payload.validate();if (msg != null){await context.Response.WriteAsync(new UploadResult(){msg = msg}.toJson());return;}//特定的配置var appConfig = _config.GetAppConfig(payload.app);//跨域context.Request.Headers.TryGetValue("Origin", out var origins);var origin = origins.ToString();if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)){context.Response.Headers.Add("Access-Control-Allow-Origin", origin);}//获取上传的文件分片var file = context.Request.Form.Files.FirstOrDefault();if (file == null || file.Length == 0){await context.Response.WriteAsync(new UploadResult(){msg = "There is no file data"}.toJson());return;}//后缀验证var ext = Path.GetExtension(file.FileName);if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)){await context.Response.WriteAsync(new UploadResult(){msg = "File extension is not allowed"}.toJson());return;}//获取参数getParams(context, out var chunkNumber, out var chunkSize, out var totalSize, out string identifier,out string filename, out int totalChunks);//验证参数var validMsg = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, file.Length, totalChunks, payload.GetByteSize() ?? _config.GetByteSize());if (validMsg != null){await context.Response.WriteAsync(new UploadResult(){msg = validMsg}.toJson());return;}else{var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifier);try{using (var fileStream = File.OpenWrite(chunkFilename)){var stream = file.OpenReadStream();stream.CopyTo(fileStream);fileStream.Flush(true);countDict.AddOrUpdate(identifier, 1, (key, oldValue) => oldValue + 1);}if (chunkNumber == totalChunks){//验证块的完整性while (true){if (countDict.GetValueOrDefault(identifier) < totalChunks){await Task.Delay(TimeSpan.FromMilliseconds(500));}else{countDict.Remove(identifier, out _);break;}}//merge file;string[] chunkFiles = Directory.GetFiles(Path.Combine(_config.PhysicalPath, temporaryFolder),"uploader-" + identifier + ".*",SearchOption.TopDirectoryOnly);var fileUrl = await MergeChunkFiles(payload, ext, chunkFiles);await context.Response.WriteAsync(new UploadResult(){ok = true,url = fileUrl}.toJson());}else{await context.Response.WriteAsync("partly_done");return;}}catch (Exception exp){await context.Response.WriteAsync(new UploadResult(){msg = exp.Message}.toJson());return;}}}catch (TokenExpiredException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has expired"}.toJson());}catch (SignatureVerificationException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has invalid signature"}.toJson());}}else{context.Response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;await context.Response.WriteAsync($"Request method '{context.Request.Method}' is not supported");}}
4、上传结果
上传成功
{"ok":true,"msg":null,"url":"http://localhost:6001/test/2019/06/17/abcd.jpg"}
上传失败
{"ok":false,"msg":"The file is too big","url":null}
Gitee：https://gitee.com/loogn/UploadServer