minio文件存储

参考

安装与部署

springboot整合minio

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.6.7</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.zzhua</groupId>
    <artifactId>demo-minio</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <!-- 在spring-boot-dependencies-2.6.7中限制了版本为3.14.9, 这与minio中依赖的版本不一致 -->
        <okhttp3.version>4.8.1</okhttp3.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-logging</artifactId>
        </dependency>
        <dependency>
            <groupId>io.minio</groupId>
            <artifactId>minio</artifactId>
            <version>8.4.5</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-configuration-processor</artifactId>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.68</version>
        </dependency>
    </dependencies>


</project>

application.yml

server:
  port: 9090
minio:
  accessKey: ~~~
  secretKey: ~~~
  endpoint: http://119.23.61.24:9000
  bucket: zzhua-bucket
  uploadUrl: ${minio.endpoint}/${minio.bucket}

MinioProperties

@Data
@ConfigurationProperties(prefix = "minio")
public class MinioProperties {

    private String accessKey;

    private String secretKey;

    private String endpoint;

    private String bucket;

    private String uploadUrl;

}

MinioConfig

@Configuration
@EnableConfigurationProperties(MinioProperties.class)
public class MinioConfig {

    @Autowired
    private MinioProperties minioProperties;

    @Bean
    public MinioClient minioClient() {
        MinioClient minioClient = MinioClient.builder()
                .credentials(minioProperties.getAccessKey(), minioProperties.getSecretKey())
                .endpoint(minioProperties.getEndpoint())
                .build();
        return minioClient;
    }

}

MinioApp

@SpringBootApplication
public class MinioApp {

    public static void main(String[] args) {
        SpringApplication.run(MinioApp.class, args);
    }

}

测试基本功能

bucket是否存在

@SpringBootTest(classes = MinioApp.class)
public class TestMinioApp {

    @Autowired
    private MinioClient minioClient;
	
	@Test
	public void testBucketExists() throws Exception {
	     boolean b = minioClient.bucketExists(BucketExistsArgs.builder().bucket("test-bucket").build());
	     System.out.println(b);
	}

}

创建bucket

@Test
public void testMakeBucket() throws Exception {
    minioClient.makeBucket(MakeBucketArgs.builder().bucket("zzhua-bucket").build());
}

修改bucket的访问权限

@Test
public void testSetBucketPolicy() throws Exception {

    StringBuilder builder = new StringBuilder();
    builder.append("{\n");
    builder.append("    \"Statement\": [\n");
    builder.append("        {\n");
    builder.append("            \"Action\": [\n");
    builder.append("                \"s3:GetBucketLocation\",\n");
    builder.append("                \"s3:ListBucket\"\n");
    builder.append("            ],\n");
    builder.append("            \"Effect\": \"Allow\",\n");
    builder.append("            \"Principal\": \"*\",\n");
    builder.append("            \"Resource\": \"arn:aws:s3:::zzhua-bucket\"\n");
    builder.append("        },\n");
    builder.append("        {\n");
    builder.append("            \"Action\": \"s3:GetObject\",\n");
    builder.append("            \"Effect\": \"Allow\",\n");
    builder.append("            \"Principal\": \"*\",\n");
    builder.append("            \"Resource\": \"arn:aws:s3:::zzhua-bucket/myobject*\"\n");
    builder.append("        }\n");
    builder.append("    ],\n");
    builder.append("    \"Version\": \"2012-10-17\"\n");
    builder.append("}\n");
    minioClient.setBucketPolicy(SetBucketPolicyArgs.builder().bucket("zzhua-bucket").config(builder.toString()).build());
}

查询所有的bucket

@Test
public void testListBucket() throws Exception {
    List<Bucket> buckets = minioClient.listBuckets();
    for (Bucket bucket : buckets) {
        System.out.println(bucket.name() + bucket.creationDate().toString());
    }
}

删除指定的bucket

@Test
public void testRemoveBucket() throws Exception {
    minioClient.removeBucket(RemoveBucketArgs.builder().bucket("test-bucket").build());
}

上传文件到minio

@Test
public void testPutObject() throws Exception {

    File file = new File("C:\\Users\\zzhua195\\Desktop\\soft-dev.png");

    ObjectWriteResponse objectWriteResponse = minioClient.putObject(PutObjectArgs.builder()
            .bucket("zzhua-bucket")
            .object(file.getName())
            .stream(new FileInputStream(file), file.length(), -1)
            .build());

    System.out.println(objectWriteResponse.etag());
    System.out.println(objectWriteResponse.versionId());
    System.out.println(objectWriteResponse);

    // 也可以用minioClient.uploadObject
}

查看对象的描述信息

@Test
 public void testStatObject() throws Exception {

     StatObjectResponse statObjectResponse = minioClient.statObject(StatObjectArgs.builder()
             .bucket("zzhua-bucket")
             .object("soft-dev.png")
             .build()
     );

     System.out.println(statObjectResponse);

 }

获取文件的预签名访问地址

@Test
public void testGetPresignedObjectUrl() throws Exception {

    // bucket如果是私有的, 那么就需要生成签名的url才能访问。如果是公有的, 那就可以直接访问。
    String presignedObjectUrl = minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder()
            .bucket("zzhua-bucket")
            .object("soft-dev.png")
            .method(Method.GET)
            .expiry(180, TimeUnit.SECONDS) // 3分钟之后失效
            .build()
    );

    System.out.println(presignedObjectUrl);

}

后台获取minio文件

@Test
public void testGetObject() throws Exception{
    GetObjectResponse getObjectResponse = minioClient.getObject(GetObjectArgs.builder()
            .bucket("zzhua-bucket")
            .object("soft-dev.png")
            .build()
    );


    /*ByteArrayOutputStream baos = new ByteArrayOutputStream();

    byte[] bytes = new byte[1024];

    int len = 0;
    while ((len = object.read(bytes)) != -1) {
        baos.write(bytes, 0, len);
    }

    System.out.println(baos.toByteArray().length);*/
}

获取bucket中的所有文件

@Test
public void testListObjects() throws Exception{
    Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket("zzhua-bucket").build());

    for (Result<Item> result : results) {
        System.out.println(result.get().objectName());
    }

}

移除指定的文件

@Test
public void testRemoveObject() throws Exception{
    minioClient.removeObject(RemoveObjectArgs.builder()
            .bucket("zzhua-bucket")
            .object("soft-dev.png")
            .build()
    );
}

单文件上传

前端先向后端申请1个上传文件到minio的文件上传路径，返回将上文件上传到minio所需要携带的表单信息

可参考GetPresignedPostFormData

public class GetPresignedPostFormData {
    /**
     * MinioClient.presignedPostPolicy() example.
     */
    public static void main(String[] args)
            throws IOException, NoSuchAlgorithmException, InvalidKeyException {
        try {
            /* play.min.io for test and development. */
            MinioClient minioClient =
                    MinioClient.builder()
                            .endpoint("http://119.23.61.24:9000")
                            .credentials("~~~", "~~~")
                            .build();

            /* Amazon S3: */
            // MinioClient minioClient =
            //     MinioClient.builder()
            //         .endpoint("https://s3.amazonaws.com")
            //         .credentials("YOUR-ACCESSKEY", "YOUR-SECRETACCESSKEY")
            //         .build();

            // Create new post policy for 'my-bucketname' with 7 days expiry from now.
            PostPolicy policy = new PostPolicy("zzhua-bucket", ZonedDateTime.now().plusHours(2));

            String objectName = "test5.png";

            // Add condition that 'key' (object name) equals to 'my-objectname'.
            policy.addEqualsCondition("key", objectName);

            // Add condition that 'Content-Type' starts with 'image/'.
            policy.addStartsWithCondition("Content-Type", "image/");

            // Add condition that 'content-length-range' is between 64kiB to 10MiB.
            policy.addContentLengthRangeCondition(64 * 1024, 10 * 1024 * 1024);

            Map<String, String> formData = minioClient.getPresignedPostFormData(policy);

            // Upload an image using POST object with form-data.
            MultipartBody.Builder multipartBuilder = new MultipartBody.Builder();
            multipartBuilder.setType(MultipartBody.FORM);
            for (Map.Entry<String, String> entry : formData.entrySet()) {
                multipartBuilder.addFormDataPart(entry.getKey(), entry.getValue());
            }
            multipartBuilder.addFormDataPart("key", objectName);
            multipartBuilder.addFormDataPart("Content-Type", "image/png");

            // "file" must be added at last.
            multipartBuilder.addFormDataPart(
                    "file", "test4.png", RequestBody.create(new File("D:\\myowrk\\Notes\\java-developer-document\\知识库\\网络安全\\kali渗透\\images\\02-Metasploit-1706781339305.png"), null));

            Request request =
                    new Request.Builder()
                            .url("http://119.23.61.24:9000/zzhua-bucket")
                            .post(multipartBuilder.build())
                            .build();
            OkHttpClient httpClient = new OkHttpClient().newBuilder().build();
            Response response = httpClient.newCall(request).execute();
            if (response.isSuccessful()) {
                System.out.println("Pictures/avatar.png is uploaded successfully using POST object");
            } else {
                System.out.println("Failed to upload Pictures/avatar.png");
            }

            // Print curl command usage to upload file /tmp/userpic.jpg.
            /*System.out.print("curl -X POST ");
            for (Map.Entry<String, String> entry : formData.entrySet()) {
                System.out.print(" -F " + entry.getKey() + "=" + entry.getValue());
            }
            System.out.print(" -F key=my-objectname -F Content-Type=image/jpg");
            System.out.println(" -F file=@/tmp/userpic.jpg https://play.min.io/my-bucketname");*/
        } catch (MinioException e) {
            System.out.println("Error occurred: " + e);
        }
    }
}

UploadSingleFileController

@Slf4j
@RestController
@RequestMapping("upload")
public class UploadSingleFileController {

    @Autowired
    private MinioClient minioClient;

    @Autowired
    private MinioProperties minioProperties;

    @RequestMapping("singleFile")
    public Result singleFile(@RequestParam String filename) {

        PostPolicy postPolicy = new PostPolicy(minioProperties.getBucket(), ZonedDateTime.now().plusHours(2));

        String date = DateTimeFormatter.ofPattern("yyyyMMdd").format(LocalDate.now());

		// 这里中间有斜杠, 将会在minio中创建文件夹
        String key = date + "/" + filename;

        // key的格式: {yyyyMMdd}/{filename}
        postPolicy.addEqualsCondition("key", key);

        // 后续前端上传时, 表单中指定的key必须与此处的key完全一样
        System.out.println("key: " + key);

        try {
            // A: 前端上传时的表单需要指定:
            //      1. 这里全部的formData
            //      2. key, 值为上面policy中设置的key
            //      3. file, 值为文件
            // B: 前端上传地址为: http://ip:9000/{bucket}
            // C: 前端上传之后, 访问url为: http://ip:9000/{bucket}/{key}
            Map<String, String> formData = minioClient.getPresignedPostFormData(postPolicy);
            System.out.println(JSON.toJSONString(formData));

            FileUploadDTO uploadDTO = new FileUploadDTO();
            uploadDTO.setFileName(filename);
            uploadDTO.setFormData(formData);

            uploadDTO.setUploadUrl(minioProperties.getUploadUrl());

            return Result.ok(uploadDTO);
        } catch (Exception e) {
            e.printStackTrace();
            return Result.fail("上传失败");
        }
    }

}

分片上传

minio分片上传分为后端分片上传和前端分片上传。

后端分片上传：直接将文件进行分片，然后将各个分片都上传到minio中，然后合并文件，然后删除所有分片文件

前端分片上传：前端根据用户上传文件的大小决定是否要使用分片上传（minio设定是每个分片最小5M，除最后1个分片外，否则合并会报错），然后如果决定要分片，计算分片数量，然后请求后台 获得每个分片的上传地址，前端进行文件拆分并自行将文件上传到对应的路径，当所有的分片上传完成之后，再请求后台合并文件的接口，将文件合并成1个文件，后台删除所有分片文件。后台可以根据分片数量和根据前缀查询指定文件再minio对应的分片序号统计还有哪些分片没有上传完成来实现断点续传。

手动示例

第一步：将mysql.pdf拆分成2个分片，名为mysql_1.chunk（大小为：5242880）和mysql_2.chunk（大小为：2593246），后面模拟前端上传这2个文件

public class File2Chunk {

    public static void main(String[] args) throws Exception {

        File file = new File("C:\\Users\\zzhua195\\Desktop\\mysql.pdf");
        String fileNameNoExt = file.getName().substring(0, file.getName().lastIndexOf("."));

        RandomAccessFile raf = new RandomAccessFile(file, "rw");
        byte[] bytes1 = new byte[5 * 1024 * 1024];
        raf.read(bytes1);


        FileOutputStream fos1 = new FileOutputStream(file.getParent() + "/" + fileNameNoExt + "_1" + ".chunk");

        fos1.write(bytes1);
        fos1.flush();
        fos1.close();

        FileOutputStream fos2 = new FileOutputStream(file.getParent() + "/" + fileNameNoExt + "_2" + ".chunk");

        int len = 0;
        byte[] bytes = new byte[1024];
        while ((len = raf.read(bytes)) != -1) {
            fos2.write(bytes, 0, len);
        }
        fos2.flush();
        fos2.close();

    }

}

第二步：为每个分片获取对应的上传路径，要注意下：1. objectName就是分片名。 2.返回的url不能直接使用，要使用url解码。3. 对于minioClient.getPresignedObjectUrl(…)方法，设置不同的method有不同的作用

public class PresignedPutObject {
    /**
     * MinioClient.presignedPutObject() example.
     */
    public static void main(String[] args)
            throws IOException, NoSuchAlgorithmException, InvalidKeyException {
        try {
            /* play.min.io for test and development. */
            MinioClient minioClient =
                    MinioClient.builder()
                            .endpoint("http://119.23.61.24:9000")
                            .credentials("CHEsnxJpF42dWcglylTi", "xWfrqTbMFo0w2o5f8jSaYLW1XRmj8Bff3wErfboS")
                            .build();

            /* Amazon S3: */
            // MinioClient minioClient =
            //     MinioClient.builder()
            //         .endpoint("https://s3.amazonaws.com")
            //         .credentials("YOUR-ACCESSKEY", "YOUR-SECRETACCESSKEY")
            //         .build();

            // Get presigned URL string to upload 'my-objectname' in 'my-bucketname'
            // with response-content-type as application/json and its life time is
            // one day.
            Map<String, String> reqParams = new HashMap<String, String>();
            reqParams.put("response-content-type", "application/json");

            String url = minioClient.getPresignedObjectUrl(
                    GetPresignedObjectUrlArgs.builder()
                            .method(Method.PUT)
                            .bucket("zzhua-bucket")
                            .object("mysql_1.chunk")
                            .expiry(60 * 60 * 24)
                            .extraQueryParams(reqParams)
                            .build());
            // System.out.println(url);

            // 注意要把这个url解码了才能正常使用
            // 然后将文件上传到这个地址上
            System.out.println(URLDecoder.decode(url, StandardCharsets.UTF_8.name()));
        } catch (MinioException e) {
            System.out.println("Error occurred: " + e);
        }
    }
}

第三步：使用postman将每个分片都上传到minio（模拟前端上传分片），由于本地使用的apiPost不支持binary上传，所有使用 网页版的ApiFox

1. 直接将返回的url粘贴到路径处，签名参数会自动填充到params中
2. Body使用binary，然后选择对应的分片文件（这个看到有人用put请求方式，并且使用form-data，key为file，值为文件，未试）
3. Header中须添加Content-Length请求头，值为分片文件的大小
4. 请求方式为PUT

上传mysql_1.chunk（如下图），然后再上传mysql_2.chunk（省略）

第四步：查询在minio中mysql_前缀下的对象有多少个，然后合并这些分片成1个文件，合并成功之后删除这些分片

查询指定前缀的分片

    @Test
    void name() throws Exception {

        Iterable<Result<Item>> mysql_ = minioClient.listObjects(ListObjectsArgs.builder().bucket("zzhua-bucket").prefix("mysql_").build());
        for (Result<Item> itemResult : mysql_) {
            System.out.println(itemResult.get().objectName());
        }

    }

合并文件

public class ComposeObject {
    /**
     * MinioClient.composeObject() example.
     */
    public static void main(String[] args)
            throws IOException, NoSuchAlgorithmException, InvalidKeyException {
        try {
            /* play.min.io for test and development. */
            MinioClient minioClient =
                    MinioClient.builder()
                            .endpoint("http://119.23.61.24:9000")
                            .credentials("~~~", "~~~")
                            .build();

            /* Amazon S3: */
            // MinioClient minioClient =
            //     MinioClient.builder()
            //         .endpoint("https://s3.amazonaws.com")
            //         .credentials("YOUR-ACCESSKEY", "YOUR-SECRETACCESSKEY")
            //         .build();

            // Create a ComposeSource to compose Object.
            List<ComposeSource> sources = new ArrayList<ComposeSource>();
            sources.add(
                    ComposeSource.builder()
                            .bucket("zzhua-bucket")
                            .object("mysql_1.chunk")
                            .build());
            sources.add(
                    ComposeSource.builder()
                            .bucket("zzhua-bucket")
                            .object("mysql_2.chunk")
                            .build());

            minioClient.composeObject(
                    ComposeObjectArgs.builder()
                            .bucket("zzhua-bucket")
                            .object("mysql.pdf")
                            .sources(sources)
                            .build());
            System.out.println("Object Composed successfully");

      /*{
        ServerSideEncryptionCustomerKey srcSsec =
            new ServerSideEncryptionCustomerKey(
                new SecretKeySpec(
                    "01234567890123456789012345678901".getBytes(StandardCharsets.UTF_8), "AES"));

        ServerSideEncryption sse =
            new ServerSideEncryptionCustomerKey(
                new SecretKeySpec(
                    "12345678912345678912345678912345".getBytes(StandardCharsets.UTF_8), "AES"));

        List<ComposeSource> sources = new ArrayList<ComposeSource>();
        sources.add(
            ComposeSource.builder()
                .bucket("my-bucketname")
                .object("my-objectname-one")
                .ssec(srcSsec)
                .build());
        sources.add(
            ComposeSource.builder()
                .bucket("my-bucketname")
                .object("my-objectname-two")
                .ssec(srcSsec)
                .build());

        minioClient.composeObject(
            ComposeObjectArgs.builder()
                .bucket("my-destination-bucket")
                .object("my-destination-object")
                .sources(sources)
                .sse(sse)
                .build());
        System.out.println("Object Composed successfully");
      }*/

        } catch (MinioException e) {
            System.out.println("Error occurred: " + e);
        }
    }


}