java 亚马逊如何采集商品 java调用最新亚马逊api

转载

mob64ca141275de 2024-07-05 06:51:04

文章标签 java 亚马逊如何采集商品 aws 云计算 s3 java 文章分类 Java 后端开发

amazon (S3) 是一个公开的服务，Web 应用程序开发人员可以使用它存储数字资产，包括图片、视频、音乐和文档等。除去aws本身的服务可以将一些内容直接存储到s3桶之外，在很多情况下，还需要使用s3桶和外部服务进行交互，比如下面一些场景：

场景一：配置使用aws服务产生的账单费用以文件的形式定期发送到s3桶，本地服务获取这些文件，生成新的账单或进行费用监控。
场景二：配置使用aws服务产生的日志文件定期发送到s3桶，本地服务获取这些文件，做数据分析，预防风险。
场景三：将s3桶当做无限量永久文件存储器，定期将文件上传到s3桶。

无论哪种场景，使用控制台手动同步这些文件显然不是明智的选择，那么就需要用到Amazon提供的各种API来实现这些操作。下面是使用java对s3桶操作的一些示例，需要的小伙伴可以使用cv大法啦。
首先引入java版的sdk。如果是本地开发，一般可以只引入aws-java-sdk一个jar包，它包含了所有aws服务的包。而通常我们只使用aws服务的一种或几种，那么我们可以单独引入使用的相关服务的包。其中kms和core是所有服务包的依赖包，为减少因版本可能带来的问题，建议也进行显式引入。这里我选择的版本是：1.11.538。

pom.xml

<!-- aws -->
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-core</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-kms</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-s3</artifactId>
    <version>${awsjavasdk.version}</version>
    <scope>compile</scope>
</dependency>

连接awsAPI基本信息类，替换其中的ak、sk和区域。

AwsBaseClient

package com.study.awsbase;
 
import com.amazonaws.regions.Regions;
 
/**
 * 连接aws的api的基本信息
 *
 * @author guoj
 * @date 2019年10月8日 10:32:17
 */
public class AwsBaseClient {
 
    /**
     * Access key ID
     */
    protected String awsAccessKey = "ak";
    /**
     * Secret access key
     */
    protected String awsSecretKey = "sk";
    /**
     * 区域
     */
    protected Regions regions = Regions.CN_NORTH_1;
 
    public AwsBaseClient() {
    }
 
    public AwsBaseClient(String awsAccessKey, String awsSecretKey, Regions regions) {
        this.awsAccessKey = awsAccessKey;
        this.awsSecretKey = awsSecretKey;
        this.regions = regions;
    }
 
    public void setBaseClient(String awsAccessKey, String awsSecretKey, Regions regions) {
        this.awsAccessKey = awsAccessKey;
        this.awsSecretKey = awsSecretKey;
        this.regions = regions;
    }
 
    public AwsBaseClient withAwsAccessKey(String awsAccessKey) {
        this.awsAccessKey = awsAccessKey;
        return this;
    }
 
    public AwsBaseClient withAwsSecretKey(String awsSecretKey) {
        this.awsSecretKey = awsSecretKey;
        return this;
    }
 
    public AwsBaseClient withRegions(Regions regions) {
        this.regions = regions;
        return this;
    }
}

获取连接s3桶客户端的类。这里提供了供子类获取默认s3客户端方法，供子类获取自定义s3客户端的方法，代外部类获取s3客户端的默认以自定义方法。

AwsS3Client

package com.study.s3;
 
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.study.awsbase.AwsBaseClient;
 
/**
 * 连接s3
 *
 * @author guoj
 * @date 2019年10月8日 10:32:17
 */
public class AwsS3Client extends AwsBaseClient {
 
    protected AmazonS3 s3;
 
    public AwsS3Client() {
        super();
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
        this.s3 = AmazonS3ClientBuilder.standard().withRegion(regions).withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build();
    }
 
    public AwsS3Client(String awsAccessKey, String awsSecretKey, Regions regions) {
        super(awsAccessKey, awsSecretKey, regions);
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
        this.s3 = AmazonS3ClientBuilder.standard().withRegion(regions).withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build();
    }
 
    public void setS3Client(String awsAccessKey, String awsSecretKey, Regions regions) {
        setBaseClient(awsAccessKey, awsSecretKey, regions);
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
        this.s3 = AmazonS3ClientBuilder.standard().withRegion(regions).withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build();
    }
 
    public AmazonS3 getS3Client() {
        return s3;
    }
 
    public AmazonS3 getS3Client(String awsAccessKey, String awsSecretKey, Regions regions) {
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
        s3 = AmazonS3ClientBuilder.standard().withRegion(regions).withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build();
        return s3;
    }
 
}

s3桶常用操作类。

AwsS3Bucket

package com.study.s3;
 
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.model.analytics.AnalyticsConfiguration;
 
import java.util.ArrayList;
import java.util.List;
 
/**
 * s3桶常用操作
 *
 * @author guoj
 * @date 2019年10月8日 11:06:46
 */
public class AwsS3Bucket extends AwsS3Client {
 
    /**
     * s3桶名称
     * 全球唯一，如果是中国区域，则中国区域唯一
     */
    private String bucketName = "sinnet-test-bucket-001";
    /**
     * s3区域
     */
    private String region = Regions.CN_NORTH_1.getName();
 
    /**
     * 列出所有的s3桶
     */
    public void listBuckets() {
        List<Bucket> buckets = s3.listBuckets();
        buckets.forEach(item -> {
            System.out.println(item.toString());
        });
    }
 
    /**
     * 创建一个s3桶-默认
     */
    public void createBucketBase() {
        Bucket bucket = s3.createBucket(bucketName);
        System.out.println(bucket.toString());
    }
 
    /**
     * 创建一个s3桶-带参数
     */
    public void createBucketWithParams() {
        //指定名称和区域
        CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName, region);
        //是否启用对象锁-启用后，阻止删除对象
        createBucketRequest.setObjectLockEnabledForBucket(true);
        Bucket bucket = s3.createBucket(createBucketRequest);
        System.out.println(bucket.toString());
    }
 
    /**
     * 删除一个s3桶
     */
    public void deleteBucket() {
        DeleteBucketRequest deleteBucketRequest = new DeleteBucketRequest(bucketName);
        s3.deleteBucket(deleteBucketRequest);
        System.out.println("delete bucket success");
    }
 
    /**
     * s3桶配置
     */
    public void configBucket() {
        /**
         * 加速配置
         */
        BucketAccelerateConfiguration bucketAccelerateConfiguration = new BucketAccelerateConfiguration(BucketAccelerateStatus.Enabled);
        s3.setBucketAccelerateConfiguration(bucketName, bucketAccelerateConfiguration);
        /**
         * 权限配置
         */
        //公共访问权限
        SetBucketAclRequest setBucketAclRequest = new SetBucketAclRequest(bucketName, CannedAccessControlList.Private);
        s3.setBucketAcl(setBucketAclRequest);
        //访问控制列表
        AccessControlList accessControlList = new AccessControlList();
        accessControlList.setRequesterCharged(true);
        accessControlList.setOwner(null);
        SetBucketAclRequest setBucketAclRequest2 = new SetBucketAclRequest(bucketName, accessControlList);
        s3.setBucketAcl(setBucketAclRequest2);
        /**
         * 分析配置
         */
        AnalyticsConfiguration analyticsConfiguration = new AnalyticsConfiguration();
        analyticsConfiguration.setId(null);
        SetBucketAnalyticsConfigurationRequest setBucketAnalyticsConfigurationRequest = new SetBucketAnalyticsConfigurationRequest(bucketName, analyticsConfiguration);
        s3.setBucketAnalyticsConfiguration(setBucketAnalyticsConfigurationRequest);
        /**
         * 生命周期配置
         */
        BucketLifecycleConfiguration bucketLifecycleConfiguration = new BucketLifecycleConfiguration();
        List<BucketLifecycleConfiguration.Rule> rules = new ArrayList();
        //需要预先制定规则
        BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule().withId(null);
        rules.add(rule);
        bucketLifecycleConfiguration.setRules(rules);
        SetBucketLifecycleConfigurationRequest setBucketLifecycleConfigurationRequest = new SetBucketLifecycleConfigurationRequest(bucketName, bucketLifecycleConfiguration);
        s3.setBucketLifecycleConfiguration(setBucketLifecycleConfigurationRequest);
        /**
         * 加密配置
         * 当对象存储在s3时默认是加密的
         */
        SetBucketEncryptionRequest setBucketEncryptionRequest = new SetBucketEncryptionRequest();
        setBucketEncryptionRequest.setBucketName(bucketName);
        ServerSideEncryptionConfiguration serverSideEncryptionConfiguration = new ServerSideEncryptionConfiguration();
        //同样，需要预先制定规则
        serverSideEncryptionConfiguration.setRules(null);
        setBucketEncryptionRequest.setServerSideEncryptionConfiguration(serverSideEncryptionConfiguration);
        s3.setBucketEncryption(setBucketEncryptionRequest);
        /**
         * 版本控制配置
         */
        BucketVersioningConfiguration bucketVersioningConfiguration = new BucketVersioningConfiguration();
        bucketVersioningConfiguration.setMfaDeleteEnabled(true);
        bucketVersioningConfiguration.setStatus(BucketVersioningConfiguration.ENABLED);
        SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = new SetBucketVersioningConfigurationRequest(bucketName, bucketVersioningConfiguration);
        s3.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest);
 
        /**
         * 为s3指定一个策略-s3的策略是唯一的
         */
        s3.setBucketPolicy(null);
        /**
         * 日志记录配置
         */
        s3.setBucketLoggingConfiguration(null);
        /**
         * 通知配置
         */
        s3.setBucketNotificationConfiguration(null);
        /**
         * 复制配置
         */
        s3.setBucketReplicationConfiguration(null);
        /**
         * 标签配置
         */
        s3.setBucketTaggingConfiguration(null);
        /**
         * 静态网站托管配置
         */
        s3.setBucketWebsiteConfiguration(null);
        /**
         * 指标配置
         */
        s3.setBucketMetricsConfiguration(null);
 
    }
 
}

s3桶文件操作类：

AwsS3File

package com.study.s3;
 
import com.amazonaws.services.s3.model.*;
 
import java.io.*;
import java.util.List;
 
/**
 * s3桶文件的常用操作
 *
 * @author guoj
 * @date 2019年10月8日 14:30:36
 */
public class AwsS3File extends AwsS3Client {
 
    /**
     * s3桶名称
     */
    private String bucketName = "sinnet-test-bucket-001";
    /**
     * 上传到s3桶后的文件名
     */
    private String key = "test-folder/sinnet-test-file.txt";
    /**
     * 本地文件
     */
    private String localFilePath = "e:/test/s3/";
    private String localFileName = "1.txt";
    private String localFilePathName = localFilePath + localFileName;
 
    /**
     * 列出一个s3桶内的文件
     */
    public void listFiles() {
        //可以加入文件名前缀当作搜索条件
        String prefix = null;
        ObjectListing objectListing = s3.listObjects(bucketName, prefix);
        List<S3ObjectSummary> s3ObjectSummaryList = objectListing.getObjectSummaries();
        s3ObjectSummaryList.forEach(item -> {
            //文件路径及名称-如果是文件夹，以"/"标识路径
            System.out.print(item.getKey() + ",");
            //文件UUID
            System.out.println(item.getETag());
        });
    }
 
    /**
     * 上传一个文件到s3桶
     * 注意1：如果key相同，则替换
     * 注意2：如果key是一个层次路径，那么s3会自动创建相应文件夹
     */
    public void uploadFile() {
        File file = new File(localFilePathName);
        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, file);
        //设置权限属性等-非必需
//        putObjectRequest.setCannedAcl(null);
        //上传
        PutObjectResult putObjectResult = s3.putObject(putObjectRequest);
        System.out.println(putObjectResult.getETag());
    }
 
    /**
     * 从s3桶删除一个文件
     * 注意：如果删除的是最后一个文件，那么，上层文件夹也会被同时删除
     */
    public void deleteFile() {
        s3.deleteObject(bucketName, key);
    }
 
    /**
     * 获取一个文件
     *
     * @throws IOException
     */
    public S3Object downloadFile() throws IOException {
        //获取文件
        S3Object s3Object = s3.getObject(bucketName, key);
        //s3文件输入流，继承了FilterInputStream
        S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent();
        //文件属性等数据
        ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
        //文件的key-可以用来获取文件名
        System.out.println(s3Object.getKey());
        //桶名
        System.out.println(s3Object.getBucketName());
        System.out.println(s3Object.getRedirectLocation());
        System.out.println(s3Object.getTaggingCount());
        return s3Object;
    }
 
    /**
     * 获取一个文件,并打印出来
     *
     * @throws IOException
     */
    public void downloadFileWithPrint() throws IOException {
        S3Object s3Object = s3.getObject(bucketName, key);
        S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent();
        //如果是txt文件，可以把内容打印出来
        StringBuffer stringBuffer = new StringBuffer();
        byte[] buffer = new byte[1024];
        while ((s3ObjectInputStream.read(buffer)) != -1) {
            stringBuffer.append(new String(buffer, "UTF-8"));
        }
        s3ObjectInputStream.close();
        System.out.println(stringBuffer.toString().trim());
    }
 
    /**
     * 获取一个文件，并保存到本地
     *
     * @throws IOException
     */
    public void downloadFileWithSave() throws IOException {
        S3Object s3Object = s3.getObject(bucketName, key);
        S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent();
        OutputStream os = null;
        try {
            //文件名
            String fileName = s3Object.getKey().substring(s3Object.getKey().lastIndexOf("/"));
            os = new FileOutputStream(new File(localFilePath + fileName));
            byte[] buffer = new byte[1024];
            while ((s3ObjectInputStream.read(buffer)) != -1) {
                os.write(buffer);
            }
            os.flush();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (os != null) {
                os.close();
            }
            if (s3ObjectInputStream != null) {
                s3ObjectInputStream.close();
            }
        }
    }
 
    /**
     * 复制一个文件
     * 比较简单，就是复制，可以选择原文件的版本
     */
    public void copyFile() {
        String sourceBucketName = bucketName;
        String sourceKey = key;
        String sourceVersionId = null;
        String destinationBucketName = bucketName;
        String destinationKey = key;
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucketName, sourceKey, sourceVersionId, destinationBucketName, destinationKey);
        CopyObjectResult copyObjectResult = s3.copyObject(copyObjectRequest);
        System.out.println(copyObjectResult.toString());
    }
 
}

另外附上两个test类：

AwsS3BucketTest

package com.study.s3;

import org.junit.Test;


public class AwsS3BucketTest {

    private AwsS3Bucket awsS3Bucket = new AwsS3Bucket();

    @Test
    public void listBuckets() {
        awsS3Bucket.listBuckets();
    }

    @Test
    public void createBucketBase() {
        awsS3Bucket.createBucketBase();
    }

    @Test
    public void createBucketWithParams() {
    }

    @Test
    public void deleteBucket() {
        awsS3Bucket.deleteBucket();
    }

    @Test
    public void configBucket() {
        awsS3Bucket.configBucket();
    }
}

AwsS3FileTest

package com.study.s3;

import org.junit.Test;

import java.io.IOException;


public class AwsS3FileTest {

    private AwsS3File awsS3File = new AwsS3File();

    @Test
    public void listFiles() {
        awsS3File.listFiles();
    }

    @Test
    public void uploadFile() {
        awsS3File.uploadFile();
    }

    @Test
    public void deleteFile() {
    }

    @Test
    public void downloadFile() {
    }

    @Test
    public void downloadFileWithPrint() {
        try {
            awsS3File.downloadFileWithPrint();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test
    public void downloadFileWithSave() {
        try {
            awsS3File.downloadFileWithSave();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test
    public void copyFile() {
    }
}

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。