diff --git a/.gitignore b/.gitignore index 631ffd7da..8279d7899 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ site/ /.idea/ /Minio.iml **/access.log -build/ vendor/**/*.js vendor/**/*.json release @@ -23,5 +22,5 @@ prime/ stage/ .sia_temp/ config.json -healthcheck -check-user +dockerscripts/check-user +dockerscripts/healthcheck \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index a0e8dac0e..9387386c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,14 @@ go_import_path: github.com/minio/minio language: go +addons: + apt: + packages: + - shellcheck + +services: + - docker + # this ensures PRs based on a local branch are not built twice # the downside is that a PR targeting a different branch is not built # but as a workaround you can add the branch to this list @@ -26,7 +34,7 @@ matrix: - $HOME/gopath/pkg/mod - $HOME/go/pkg/mod - go: 1.12.1 + go: 1.12.5 script: - make - diff -au <(gofmt -s -d cmd) <(printf "") @@ -37,12 +45,14 @@ matrix: - make verify - make coverage - cd browser && yarn && yarn test && cd .. + - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh' + - os: windows env: - ARCH=x86_64 - CGO_ENABLED=0 - GO111MODULE=on - go: 1.12.1 + go: 1.12.5 script: - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe - bash buildscripts/go-coverage.sh diff --git a/Dockerfile.simpleci b/Dockerfile.simpleci index 9282a11e6..30bdad4be 100644 --- a/Dockerfile.simpleci +++ b/Dockerfile.simpleci @@ -33,7 +33,7 @@ USER ci RUN make RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")' RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")' -RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done +RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done RUN make verifiers RUN make crosscompile RUN make verify @@ -56,15 +56,17 @@ FROM ubuntu:16.04 COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh +COPY mint /mint ENV DEBIAN_FRONTEND noninteractive ENV LANG C.UTF-8 ENV GOROOT /usr/local/go ENV GOPATH /usr/local ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH +ENV MINT_ROOT_DIR /mint -RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \ - git clone https://github.com/minio/mint.git /mint && \ +RUN apt-get --yes update && apt-get --yes upgrade && \ + apt-get --yes --quiet install wget jq curl git dnsmasq && \ cd /mint && /mint/release.sh WORKDIR /mint diff --git a/mint/.gitignore b/mint/.gitignore new file mode 100644 index 000000000..ef999ea53 --- /dev/null +++ b/mint/.gitignore @@ -0,0 +1,17 @@ +*.test +*.jar +src/* +temp +__pycache__/ +log/* +minio.test +bin/* +node_modules +# exception to the rule +!log/.gitkeep +!bin/.gitkeep +*.class +*~ +run/core/minio-dotnet/bin/* +run/core/minio-dotnet/obj/* +run/core/minio-dotnet/out/* diff --git a/mint/Dockerfile b/mint/Dockerfile new file mode 100644 index 000000000..3896cf457 --- /dev/null +++ b/mint/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND noninteractive + +ENV LANG C.UTF-8 + +ENV GOROOT /usr/local/go + +ENV GOPATH /usr/local + +ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH + +RUN apt-get --yes update && apt-get --yes upgrade && \ + apt-get --yes --quiet install wget jq curl git dnsmasq && \ + git clone https://github.com/minio/minio.git /minio && \ + ln -sf /minio/mint /mint && /mint/release.sh + +WORKDIR /mint + +ENTRYPOINT ["/mint/entrypoint.sh"] diff --git a/mint/Dockerfile.dev b/mint/Dockerfile.dev new file mode 100644 index 000000000..fd98d0cd9 --- /dev/null +++ b/mint/Dockerfile.dev @@ -0,0 +1,84 @@ +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND noninteractive + +ENV LANG C.UTF-8 + +ENV GOROOT /usr/local/go + +ENV GOPATH /usr/local + +ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH + +WORKDIR /mint + +RUN apt-get --yes update && apt-get --yes upgrade && \ + apt-get --yes --quiet install wget jq curl dnsmasq + +ENV MINT_ROOT_DIR /mint +ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core +ENV MINT_RUN_SECURITY_DIR $MINT_ROOT_DIR/run/security +ENV WGET "wget --quiet --no-check-certificate" + +COPY create-data-files.sh /mint +RUN /mint/create-data-files.sh + +COPY install-packages.list /mint +COPY preinstall.sh /mint +RUN /mint/preinstall.sh + +COPY run /mint/run + +COPY build/awscli /mint/build/awscli +RUN build/awscli/install.sh + +COPY build/aws-sdk-java /mint/build/aws-sdk-java +RUN build/aws-sdk-java/install.sh + +COPY build/aws-sdk-go /mint/build/aws-sdk-go +RUN build/aws-sdk-go/install.sh + +COPY build/aws-sdk-php /mint/build/aws-sdk-php +RUN build/aws-sdk-php/install.sh + +COPY build/aws-sdk-ruby /mint/build/aws-sdk-ruby +RUN build/aws-sdk-ruby/install.sh + +COPY build/mc /mint/build/mc +RUN build/mc/install.sh + +COPY build/minio-go /mint/build/minio-go +RUN build/minio-go/install.sh + +COPY build/minio-java /mint/build/minio-java +RUN build/minio-java/install.sh + +COPY build/minio-js /mint/build/minio-js +RUN build/minio-js/install.sh + +COPY build/minio-py /mint/build/minio-py +RUN build/minio-py/install.sh + +COPY build/s3cmd /mint/build/s3cmd +RUN build/s3cmd/install.sh + +COPY build/minio-dotnet/ /mint/build/minio-dotnet/ +RUN /mint/build/minio-dotnet/install.sh + +COPY build/security /mint/build/security +RUN build/security/install.sh + +COPY build/worm /mint/build/worm +RUN build/worm/install.sh + +COPY build/healthcheck /mint/build/healthcheck +RUN build/healthcheck/install.sh + +COPY remove-packages.list /mint +COPY postinstall.sh /mint +RUN /mint/postinstall.sh + +COPY mint.sh /mint/mint.sh +COPY entrypoint.sh /mint/entrypoint.sh + +ENTRYPOINT ["/mint/entrypoint.sh"] diff --git a/mint/README.md b/mint/README.md new file mode 100644 index 000000000..6f3d1a11c --- /dev/null +++ b/mint/README.md @@ -0,0 +1,123 @@ +# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) + +Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. + +- awscli +- aws-sdk-go +- aws-sdk-php +- aws-sdk-ruby +- aws-sdk-java +- mc +- minio-go +- minio-java +- minio-js +- minio-py +- minio-dotnet +- s3cmd +- worm + +## Running Mint + +Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/). + +To run Mint with Minio Play server as test target, + +```sh +$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ + -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint +``` + +After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example +```sh +docker cp :/mint/log /tmp/logs +``` + +### Mint environment variables + +Below environment variables are required to be passed to the docker container. Supported environment variables: + +| Environment variable | Description | Example | +|:--- |:--- |:--- | +| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | +| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` | +| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | +| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | +| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full` or `worm`. Defaults to `core` | `full` | +| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | +| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | + + +### Test virtual style access against Minio server + +To test Minio server virtual style access with Mint, follow these steps: + +- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. +- Start Minio server. +- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command +```sh +$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ + -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ + -e "ENABLE_VIRTUAL_STYLE=1" minio/mint +``` + +### Mint log format + +All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. + +| JSON field | Type | Description | Example | +|:--- |:--- |:--- |:--- | +| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` | +| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` | +| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` | +| `duration` | _int_ | Time taken in milliseconds to run the test | `384` | +| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` | +| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` | +| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` | +| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` | + +## For Developers + +### Running Mint development code + +After making changes to Mint source code a local docker image can be built/run by + +```sh +$ docker build -t minio/mint . -f Dockerfile.dev +$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ + -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ + -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest +``` +In case of Worm Mode, start your server with configuration `MINT_MODE` set to `worm`. +Build/Run of local docker image for Worm mode, is to be tested against your server configuration, by + +``` +### Adding tests with new tool/SDK + +Below are the steps need to be followed + +* Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories. +* Create `install.sh` which does installation of required tool/SDK under app directory. +* Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list). +* Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image. +* Add `run.sh` in app directory under `run/core` which execute actual tests. + +#### Test data + +Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory. + +| File name | Size | +|:--- |:--- | +| datafile-0-b | 0B | +| datafile-1-b | 1B | +| datafile-1-kB |1KiB | +| datafile-10-kB |10KiB | +| datafile-33-kB |33KiB | +| datafile-100-kB |100KiB | +| datafile-1-MB |1MiB | +| datafile-1.03-MB |1.03MiB | +| datafile-5-MB |5MiB | +| datafile-6-MB |6MiB | +| datafile-10-MB |10MiB | +| datafile-11-MB |11MiB | +| datafile-65-MB |65MiB | +| datafile-129-MB |129MiB | diff --git a/mint/build/aws-sdk-go/install.sh b/mint/build/aws-sdk-go/install.sh new file mode 100755 index 000000000..d73162840 --- /dev/null +++ b/mint/build/aws-sdk-go/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" +GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go" diff --git a/mint/build/aws-sdk-java/build.xml b/mint/build/aws-sdk-java/build.xml new file mode 100644 index 000000000..312c54c1f --- /dev/null +++ b/mint/build/aws-sdk-java/build.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mint/build/aws-sdk-java/install.sh b/mint/build/aws-sdk-java/install.sh new file mode 100755 index 000000000..c3206d42a --- /dev/null +++ b/mint/build/aws-sdk-java/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java" + +cd "$(dirname "$(realpath "$0")")" + +ant init-ivy && \ + ant resolve && \ + ant compile && \ + ant jar + +cp build/jar/FunctionalTests.jar "$test_run_dir/" + +rm -rf lib/ build/ + diff --git a/mint/build/aws-sdk-java/ivy.xml b/mint/build/aws-sdk-java/ivy.xml new file mode 100644 index 000000000..115c593f3 --- /dev/null +++ b/mint/build/aws-sdk-java/ivy.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/mint/build/aws-sdk-java/src/FunctionalTests.java b/mint/build/aws-sdk-java/src/FunctionalTests.java new file mode 100644 index 000000000..2005a7807 --- /dev/null +++ b/mint/build/aws-sdk-java/src/FunctionalTests.java @@ -0,0 +1,632 @@ +/* +* Mint, (C) 2018 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package io.minio.awssdk.tests; + +import java.io.*; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; + +import java.security.*; +import java.util.*; + +import java.nio.file.*; +import java.math.BigInteger; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.SSECustomerKey; + +// Main Testing class +public class FunctionalTests { + + private static final String PASS = "PASS"; + private static final String FAILED = "FAIL"; + private static final String IGNORED = "NA"; + + private static String accessKey; + private static String secretKey; + private static String region; + private static String endpoint; + private static boolean enableHTTPS; + + private static final Random random = new Random(new SecureRandom().nextLong()); + private static String bucketName = getRandomName(); + private static boolean mintEnv = false; + + private static String file1Kb; + private static String file1Mb; + private static String file6Mb; + + private static SSECustomerKey sseKey1; + private static SSECustomerKey sseKey2; + private static SSECustomerKey sseKey3; + + private static AmazonS3 s3Client; + private static S3TestUtils s3TestUtils; + + public static String getRandomName() { + return "aws-java-sdk-test-" + new BigInteger(32, random).toString(32); + } + + /** + * Prints a success log entry in JSON format. + */ + public static void mintSuccessLog(String function, String args, long startTime) { + if (mintEnv) { + System.out.println( + new MintLogger(function, args, System.currentTimeMillis() - startTime, PASS, null, null, null)); + } + } + + /** + * Prints a failure log entry in JSON format. + */ + public static void mintFailedLog(String function, String args, long startTime, String message, String error) { + if (mintEnv) { + System.out.println(new MintLogger(function, args, System.currentTimeMillis() - startTime, FAILED, null, + message, error)); + } + } + + /** + * Prints a ignore log entry in JSON format. + */ + public static void mintIgnoredLog(String function, String args, long startTime) { + if (mintEnv) { + System.out.println( + new MintLogger(function, args, System.currentTimeMillis() - startTime, IGNORED, null, null, null)); + } + } + + public static void initTests() throws IOException { + // Create encryption key. + byte[] rawKey1 = "32byteslongsecretkeymustgenerate".getBytes(); + SecretKey secretKey1 = new SecretKeySpec(rawKey1, 0, rawKey1.length, "AES"); + sseKey1 = new SSECustomerKey(secretKey1); + + // Create new encryption key for target so it is saved using sse-c + byte[] rawKey2 = "xxbytescopysecretkeymustprovided".getBytes(); + SecretKey secretKey2 = new SecretKeySpec(rawKey2, 0, rawKey2.length, "AES"); + sseKey2 = new SSECustomerKey(secretKey2); + + // Create new encryption key for target so it is saved using sse-c + byte[] rawKey3 = "32byteslongsecretkeymustgenerat1".getBytes(); + SecretKey secretKey3 = new SecretKeySpec(rawKey3, 0, rawKey3.length, "AES"); + sseKey3 = new SSECustomerKey(secretKey3); + + // Create bucket + s3Client.createBucket(new CreateBucketRequest(bucketName)); + } + + public static void teardown() throws IOException { + + // Remove all objects under the test bucket & the bucket itself + // TODO: use multi delete API instead + ObjectListing objectListing = s3Client.listObjects(bucketName); + while (true) { + for (Iterator iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) { + S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); + s3Client.deleteObject(bucketName, summary.getKey()); + } + // more objectListing to retrieve? + if (objectListing.isTruncated()) { + objectListing = s3Client.listNextBatchOfObjects(objectListing); + } else { + break; + } + } + ; + s3Client.deleteBucket(bucketName); + } + + // Test regular object upload using encryption + public static void uploadObjectEncryption_test1() throws Exception { + if (!mintEnv) { + System.out.println( + "Test: uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + String file1KbMD5 = Utils.getFileMD5(file1Kb); + String objectName = "testobject"; + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, file1KbMD5); + mintSuccessLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb + + ", SSECustomerKey: " + sseKey1, + startTime); + } catch (Exception e) { + mintFailedLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb + + ", SSECustomerKey: " + sseKey1, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + } + + // Test downloading an object with a wrong encryption key + public static void downloadObjectEncryption_test1() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String file1KbMD5 = Utils.getFileMD5(file1Kb); + String objectName = "testobject"; + + try { + s3TestUtils.uploadObject(bucketName, "testobject", file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey2); + Exception ex = new Exception("downloadObject did not throw an S3 Access denied exception"); + mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, + startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); + throw ex; + } catch (Exception e) { + if (!e.getMessage().contains("Access Denied")) { + Exception ex = new Exception( + "downloadObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); + mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, + startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); + throw ex; + } + mintSuccessLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, + startTime); + } + } + + // Test copying object with a new different encryption key + public static void copyObjectEncryption_test1() throws Exception { + if (!mintEnv) { + System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + String file1KbMD5 = Utils.getFileMD5(file1Kb); + String objectName = "testobject"; + String dstObjectName = "dir/newobject"; + + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); + s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file1KbMD5); + } catch (Exception e) { + mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " + + sseKey2 + ", replaceDirective: " + false, + startTime); + } + + // Test copying object with wrong source encryption key + public static void copyObjectEncryption_test2() throws Exception { + if (!mintEnv) { + System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); + } + + if (!enableHTTPS) { + return; + } + + String objectName = "testobject"; + String dstObjectName = "dir/newobject"; + + long startTime = System.currentTimeMillis(); + + try { + s3TestUtils.copyObject(bucketName, objectName, sseKey3, bucketName, dstObjectName, sseKey2, false); + Exception ex = new Exception("copyObject did not throw an S3 Access denied exception"); + mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, + startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); + throw ex; + } catch (Exception e) { + if (!e.getMessage().contains("Access Denied")) { + Exception ex = new Exception( + "copyObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); + mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, + startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); + throw ex; + } + mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, + startTime); + } + } + + // Test copying multipart object + public static void copyObjectEncryption_test3() throws Exception { + if (!mintEnv) { + System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + String file6MbMD5 = Utils.getFileMD5(file6Mb); + String objectName = "testobject"; + String dstObjectName = "dir/newobject"; + + try { + s3TestUtils.uploadMultipartObject(bucketName, objectName, file6Mb, sseKey1); + s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); + s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file6MbMD5); + } catch (Exception e) { + mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " + + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " + + sseKey2 + ", replaceDirective: " + false, + startTime); + } + + // Test downloading encrypted object with Get Range, 0 -> 1024 + public static void downloadGetRangeEncryption_test1() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + String range1MD5 = Utils.getFileMD5(file1Kb); + int start = 0; + int length = 1024; + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Test downloading encrypted object with Get Range, 0 -> 1 + public static void downloadGetRangeEncryption_test2() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + int start = 0; + int length = 1; + String range1MD5 = Utils.getFileMD5(file1Kb, start, length); + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Test downloading encrypted object with Get Range, 0 -> 1024-1 + public static void downloadGetRangeEncryption_test3() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + int start = 0; + int length = 1023; + String range1MD5 = Utils.getFileMD5(file1Kb, start, length); + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Test downloading encrypted object with Get Range, 1 -> 1024-1 + public static void downloadGetRangeEncryption_test4() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + int start = 1; + int length = 1023; + String range1MD5 = Utils.getFileMD5(file1Kb, start, length); + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Test downloading encrypted object with Get Range, 64*1024 -> 64*1024 + public static void downloadGetRangeEncryption_test5() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + int start = 64 * 1024; + int length = 64 * 1024; + String range1MD5 = Utils.getFileMD5(file1Mb, start, length); + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Test downloading encrypted object with Get Range, 64*1024 -> + // 1024*1024-64*1024 + public static void downloadGetRangeEncryption_test6() throws Exception { + if (!mintEnv) { + System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); + } + + if (!enableHTTPS) { + return; + } + + long startTime = System.currentTimeMillis(); + + String objectName = "testobject"; + int start = 64 * 1024; + int length = 1024 * 1024 - 64 * 1024; + String range1MD5 = Utils.getFileMD5(file1Mb, start, length); + try { + s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); + s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); + } catch (Exception e) { + mintFailedLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); + throw e; + } + mintSuccessLog( + "downloadObjectGetRange(String bucketName, String objectName, " + + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", + "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 + + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, + startTime); + } + + // Run tests + public static void runTests() throws Exception { + + uploadObjectEncryption_test1(); + + downloadObjectEncryption_test1(); + + copyObjectEncryption_test1(); + copyObjectEncryption_test2(); + copyObjectEncryption_test3(); + + downloadGetRangeEncryption_test1(); + downloadGetRangeEncryption_test2(); + downloadGetRangeEncryption_test3(); + downloadGetRangeEncryption_test4(); + downloadGetRangeEncryption_test5(); + downloadGetRangeEncryption_test6(); + } + + public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException { + + endpoint = System.getenv("SERVER_ENDPOINT"); + accessKey = System.getenv("ACCESS_KEY"); + secretKey = System.getenv("SECRET_KEY"); + enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1"); + + region = "us-east-1"; + + if (enableHTTPS) { + endpoint = "https://" + endpoint; + } else { + endpoint = "http://" + endpoint; + } + + String dataDir = System.getenv("MINT_DATA_DIR"); + if (dataDir != null && !dataDir.equals("")) { + mintEnv = true; + file1Kb = Paths.get(dataDir, "datafile-1-kB").toString(); + file1Mb = Paths.get(dataDir, "datafile-1-MB").toString(); + file6Mb = Paths.get(dataDir, "datafile-6-MB").toString(); + } + + String mintMode = null; + if (mintEnv) { + mintMode = System.getenv("MINT_MODE"); + } + + AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + AmazonS3ClientBuilder.EndpointConfiguration endpointConfiguration = new AmazonS3ClientBuilder.EndpointConfiguration( + endpoint, region); + + AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); + clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); + clientBuilder.setEndpointConfiguration(endpointConfiguration); + clientBuilder.setPathStyleAccessEnabled(true); + + s3Client = clientBuilder.build(); + s3TestUtils = new S3TestUtils(s3Client); + + try { + initTests(); + FunctionalTests.runTests(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } finally { + teardown(); + } + } +} diff --git a/mint/build/aws-sdk-java/src/LimitedInputStream.java b/mint/build/aws-sdk-java/src/LimitedInputStream.java new file mode 100644 index 000000000..310a174e5 --- /dev/null +++ b/mint/build/aws-sdk-java/src/LimitedInputStream.java @@ -0,0 +1,60 @@ +/* +* Mint, (C) 2018 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package io.minio.awssdk.tests; + +import java.io.*; + +// LimitedInputStream wraps a regular InputStream, calling +// read() will skip some bytes as configured and will also +// return only data with configured length + +class LimitedInputStream extends InputStream { + + private int skip; + private int length; + private InputStream is; + + LimitedInputStream(InputStream is, int skip, int length) { + this.is = is; + this.skip = skip; + this.length = length; + } + + @Override + public int read() throws IOException { + int r; + while (skip > 0) { + r = is.read(); + if (r < 0) { + throw new IOException("stream ended before being able to skip all bytes"); + } + skip--; + } + if (length == 0) { + return -1; + } + r = is.read(); + if (r < 0) { + throw new IOException("stream ended before being able to read all bytes"); + } + length--; + return r; + } +} + + diff --git a/mint/build/aws-sdk-java/src/MintLogger.java b/mint/build/aws-sdk-java/src/MintLogger.java new file mode 100755 index 000000000..b6319f00a --- /dev/null +++ b/mint/build/aws-sdk-java/src/MintLogger.java @@ -0,0 +1,151 @@ +/* + * Minio Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2015, 2016, 2017, 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.awssdk.tests; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class MintLogger { + + @JsonProperty("name") + private String name; + + @JsonProperty("function") + private String function; + + @JsonProperty("args") + private String args; + + @JsonProperty("duration") + private long duration; + + @JsonProperty("status") + private String status; + + @JsonProperty("alert") + private String alert; + + @JsonProperty("message") + private String message; + + @JsonProperty("error") + private String error; + + /** + * Constructor. + **/ + public MintLogger(String function, + String args, + long duration, + String status, + String alert, + String message, + String error) { + this.name = "aws-sdk-java"; + this.function = function; + this.duration = duration; + this.args = args; + this.status = status; + this.alert = alert; + this.message = message; + this.error = error; + } + + /** + * Return JSON Log Entry. + **/ + @JsonIgnore + public String toString() { + + try { + return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + return ""; + } + + /** + * Return Alert. + **/ + @JsonIgnore + public String alert() { + return alert; + } + + /** + * Return Error. + **/ + @JsonIgnore + public String error() { + return error; + } + + /** + * Return Message. + **/ + @JsonIgnore + public String message() { + return message; + } + + /** + * Return args. + **/ + @JsonIgnore + public String args() { + return args; + } + + /** + * Return status. + **/ + @JsonIgnore + public String status() { + return status; + } + + /** + * Return name. + **/ + @JsonIgnore + public String name() { + return name; + } + + /** + * Return function. + **/ + @JsonIgnore + public String function() { + return function; + } + + /** + * Return duration. + **/ + @JsonIgnore + public long duration() { + return duration; + } +} diff --git a/mint/build/aws-sdk-java/src/S3TestUtils.java b/mint/build/aws-sdk-java/src/S3TestUtils.java new file mode 100644 index 000000000..236caf938 --- /dev/null +++ b/mint/build/aws-sdk-java/src/S3TestUtils.java @@ -0,0 +1,187 @@ +/* +* Mint, (C) 2018 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package io.minio.awssdk.tests; + +import java.io.*; +import java.util.*; +import java.nio.channels.Channels; + +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.SSECustomerKey; + +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.UploadPartRequest; + +import com.amazonaws.services.s3.model.MetadataDirective; + +import com.amazonaws.services.s3.AmazonS3; + +class S3TestUtils { + + private AmazonS3 s3Client; + + S3TestUtils(AmazonS3 s3Client) { + this.s3Client = s3Client; + } + + void uploadMultipartObject(String bucketName, String keyName, + String filePath, SSECustomerKey sseKey) throws IOException { + + File file = new File(filePath); + + List partETags = new ArrayList(); + + // Step 1: Initialize. + InitiateMultipartUploadRequest initRequest = new + InitiateMultipartUploadRequest(bucketName, keyName); + + if (sseKey != null) { + initRequest.setSSECustomerKey(sseKey); + } + + InitiateMultipartUploadResult initResponse = + s3Client.initiateMultipartUpload(initRequest); + + long contentLength = file.length(); + long partSize = 5242880; // Set part size to 5 MB. + + // Step 2: Upload parts. + long filePosition = 0; + for (int i = 1; filePosition < contentLength; i++) { + // Last part can be less than 5 MB. Adjust part size. + partSize = Math.min(partSize, (contentLength - filePosition)); + + // Create request to upload a part. + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucketName).withKey(keyName) + .withUploadId(initResponse.getUploadId()).withPartNumber(i) + .withFileOffset(filePosition) + .withFile(file) + .withPartSize(partSize); + + if (sseKey != null) { + uploadRequest.withSSECustomerKey(sseKey); + } + + // Upload part and add response to our list. + partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); + + filePosition += partSize; + } + + // Step 3: Complete. + CompleteMultipartUploadRequest compRequest = new + CompleteMultipartUploadRequest( + bucketName, + keyName, + initResponse.getUploadId(), + partETags); + + s3Client.completeMultipartUpload(compRequest); + } + + void uploadObject(String bucketName, String keyName, + String filePath, SSECustomerKey sseKey) throws IOException { + + File f = new File(filePath); + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f); + if (sseKey != null) { + putObjectRequest.withSSECustomerKey(sseKey); + } + s3Client.putObject(putObjectRequest); + } + + void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey) + throws Exception, IOException { + downloadObject(bucketName, keyName, sseKey, "", -1, -1); + } + + void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, + String expectedMD5) + throws Exception, IOException { + downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1); + } + + void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, + String expectedMD5, int start, int length) throws Exception, IOException { + GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName) + .withSSECustomerKey(sseKey); + + if (start >= 0 && length >= 0) { + getObjectRequest.setRange(start, start+length-1); + } + + S3Object s3Object = s3Client.getObject(getObjectRequest); + + int size = 0; + int c; + + S3ObjectInputStream input = s3Object.getObjectContent(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + String data = ""; + while ((c = input.read()) != -1) { + output.write((byte) c); + size++; + } + + if (length >= 0 && size != length) { + throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size); + } + + String calculatedMD5 = Utils.getBufferMD5(output.toByteArray()); + + if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) { + throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5); + + } + } + + void copyObject(String bucketName, String keyName, SSECustomerKey sseKey, + String targetBucketName, String targetKeyName, SSECustomerKey newSseKey, + boolean replace) { + CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName); + if (sseKey != null) { + copyRequest.withSourceSSECustomerKey(sseKey); + } + if (newSseKey != null) { + copyRequest.withDestinationSSECustomerKey(newSseKey); + } + if (replace) { + copyRequest.withMetadataDirective(MetadataDirective.COPY); + } + s3Client.copyObject(copyRequest); + } + + long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) { + GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName) + .withSSECustomerKey(sseKey); + ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest); + return objectMetadata.getContentLength(); + } + +} diff --git a/mint/build/aws-sdk-java/src/Utils.java b/mint/build/aws-sdk-java/src/Utils.java new file mode 100644 index 000000000..d6c971216 --- /dev/null +++ b/mint/build/aws-sdk-java/src/Utils.java @@ -0,0 +1,76 @@ +/* +* Mint, (C) 2018 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +package io.minio.awssdk.tests; + +import java.io.*; +import java.nio.channels.*; +import java.security.*; + +class Utils { + + public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception { + int numRead; + byte[] buffer = new byte[1024]; + + MessageDigest complete = MessageDigest.getInstance("MD5"); + + if (skip > -1 && length > -1) { + is = new LimitedInputStream(is, skip, length); + } + + do { + numRead = is.read(buffer); + if (numRead > 0) { + complete.update(buffer, 0, numRead); + } + } while (numRead != -1); + + return complete.digest(); + } + + public static String getInputStreamMD5(InputStream is) throws Exception { + return getInputStreamMD5(is, -1, -1); + } + + public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception { + byte[] b = createChecksum(is, start, length); + String result = ""; + + for (int i=0; i < b.length; i++) { + result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 ); + } + return result; + } + + public static String getFileMD5(String filePath) throws Exception { + return getFileMD5(filePath, -1, -1); + } + + public static String getFileMD5(String filePath, int start, int length) throws Exception { + File f = new File(filePath); + InputStream is = new FileInputStream(f); + return getInputStreamMD5(is, start, length); + } + + public static String getBufferMD5(byte[] data) throws Exception { + ByteArrayInputStream bis = new ByteArrayInputStream(data); + return getInputStreamMD5(bis); + } +} + diff --git a/mint/build/aws-sdk-php/install.sh b/mint/build/aws-sdk-php/install.sh new file mode 100755 index 000000000..1363fa0dd --- /dev/null +++ b/mint/build/aws-sdk-php/install.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php" +$WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir" +php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install diff --git a/mint/build/aws-sdk-ruby/install.sh b/mint/build/aws-sdk-ruby/install.sh new file mode 100755 index 000000000..e25186ee1 --- /dev/null +++ b/mint/build/aws-sdk-ruby/install.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +gem install --no-rdoc --no-ri aws-sdk multipart_body diff --git a/mint/build/awscli/install.sh b/mint/build/awscli/install.sh new file mode 100755 index 000000000..bd6617770 --- /dev/null +++ b/mint/build/awscli/install.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +AWS_CLI_VERSION="1.11.112" + +python -m pip install awscli==$AWS_CLI_VERSION diff --git a/mint/build/healthcheck/install.sh b/mint/build/healthcheck/install.sh new file mode 100755 index 000000000..71ffe4823 --- /dev/null +++ b/mint/build/healthcheck/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +# +# Mint (C) 2019 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/healthcheck" +GO111MODULE=on go build -o "$test_run_dir/healthcheck" "$test_run_dir/healthcheck.go" diff --git a/mint/build/mc/install.sh b/mint/build/mc/install.sh new file mode 100755 index 000000000..a41aaebdd --- /dev/null +++ b/mint/build/mc/install.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///") +if [ -z "$MC_VERSION" ]; then + echo "unable to get mc version from github" + exit 1 +fi + +test_run_dir="$MINT_RUN_CORE_DIR/mc" +$WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}" +chmod a+x "${test_run_dir}/mc" + +git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git" +(cd "$test_run_dir/mc.git"; git checkout --quiet "tags/${MC_VERSION}") +cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/" +rm -fr "$test_run_dir/mc.git" diff --git a/mint/build/minio-dotnet/install.sh b/mint/build/minio-dotnet/install.sh new file mode 100755 index 000000000..d3d92c956 --- /dev/null +++ b/mint/build/minio-dotnet/install.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet" + +MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///") +if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then + echo "unable to get minio-dotnet version from github" + exit 1 +fi + +out_dir="$MINIO_DOTNET_SDK_PATH/out" +if [ -z "$out_dir" ]; then + mkdir "$out_dir" +fi + +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/FunctionalTest.cs > "${MINIO_DOTNET_SDK_PATH}/FunctionalTest.cs" +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/MintLogger.cs > "${MINIO_DOTNET_SDK_PATH}/MintLogger.cs" +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/JsonNetLogger.cs > "${MINIO_DOTNET_SDK_PATH}/JsonNetLogger.cs" +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/Program.cs > "${MINIO_DOTNET_SDK_PATH}/Program.cs" +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/RandomStreamGenerator.cs > "${MINIO_DOTNET_SDK_PATH}/RandomStreamGenerator.cs" +curl https://raw.githubusercontent.com/minio/minio-dotnet/"${MINIO_DOTNET_SDK_VERSION}"/Minio.Functional.Tests/Minio.Functional.Tests.csproj > "${MINIO_DOTNET_SDK_PATH}/Minio.Functional.Tests.csproj" + +cd "$MINIO_DOTNET_SDK_PATH" +dotnet restore /p:Configuration=Mint +dotnet publish --runtime ubuntu.16.04-x64 --output out /p:Configuration=Mint diff --git a/mint/build/minio-go/install.sh b/mint/build/minio-go/install.sh new file mode 100755 index 000000000..5e3005199 --- /dev/null +++ b/mint/build/minio-go/install.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///") +if [ -z "$MINIO_GO_VERSION" ]; then + echo "unable to get minio-go version from github" + exit 1 +fi + +test_run_dir="$MINT_RUN_CORE_DIR/minio-go" +(git clone https://github.com/minio/minio-go && cd minio-go && git checkout --quiet "tags/$MINIO_GO_VERSION") +GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "minio-go/functional_tests.go" +rm -rf minio-go diff --git a/mint/build/minio-java/install.sh b/mint/build/minio-java/install.sh new file mode 100755 index 000000000..692190b67 --- /dev/null +++ b/mint/build/minio-java/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MINIO_JAVA_VERSION=$(curl --retry 10 -s "http://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "//{s/<.[^>]*>//g;p;q}" | sed "s/ *//g") +if [ -z "$MINIO_JAVA_VERSION" ]; then + echo "unable to get latest minio-java version from maven" + exit 1 +fi + +test_run_dir="$MINT_RUN_CORE_DIR/minio-java" +git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git" +(cd "$test_run_dir/minio-java.git"; git checkout --quiet "tags/${MINIO_JAVA_VERSION}") +$WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "http://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar" +javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "${test_run_dir}/minio-java.git/functional"/*.java +cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/" +rm -fr "$test_run_dir/minio-java.git" diff --git a/mint/build/minio-js/install.sh b/mint/build/minio-js/install.sh new file mode 100755 index 000000000..0b9c4f452 --- /dev/null +++ b/mint/build/minio-js/install.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MINIO_JS_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-js/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-js\/releases\/tag\///") +if [ -z "$MINIO_JS_VERSION" ]; then + echo "unable to get minio-js version from github" + exit 1 +fi + +test_run_dir="$MINT_RUN_CORE_DIR/minio-js" +mkdir "${test_run_dir}/test" +$WGET --output-document="${test_run_dir}/test/functional-tests.js" "https://raw.githubusercontent.com/minio/minio-js/${MINIO_JS_VERSION}/src/test/functional/functional-tests.js" +npm --prefix "$test_run_dir" install --save "minio@$MINIO_JS_VERSION" +npm --prefix "$test_run_dir" install diff --git a/mint/build/minio-py/install.sh b/mint/build/minio-py/install.sh new file mode 100755 index 000000000..8af9f98ef --- /dev/null +++ b/mint/build/minio-py/install.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///") +if [ -z "$MINIO_PY_VERSION" ]; then + echo "unable to get minio-py version from github" + exit 1 +fi + +test_run_dir="$MINT_RUN_CORE_DIR/minio-py" +# FIX https://github.com/pypa/pip/issues/5221 +python -m pip install --upgrade pip +python -m pip install --user faker +python -m pip install minio=="$MINIO_PY_VERSION" +$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" diff --git a/mint/build/s3cmd/install.sh b/mint/build/s3cmd/install.sh new file mode 100755 index 000000000..6865930c4 --- /dev/null +++ b/mint/build/s3cmd/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Always install the latest. +python -m pip install s3cmd diff --git a/mint/build/security/install.sh b/mint/build/security/install.sh new file mode 100755 index 000000000..a517bfb18 --- /dev/null +++ b/mint/build/security/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +# +# Mint (C) 2018 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/security" +GO111MODULE=on go build -o "$test_run_dir/tls-tests" "$test_run_dir/tls-tests.go" diff --git a/mint/build/worm/install.sh b/mint/build/worm/install.sh new file mode 100755 index 000000000..7cdf1fd0f --- /dev/null +++ b/mint/build/worm/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/worm" +GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/worm" "$test_run_dir/quick-worm-tests.go" diff --git a/mint/create-data-files.sh b/mint/create-data-files.sh new file mode 100755 index 000000000..ba0ce43c3 --- /dev/null +++ b/mint/create-data-files.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MINT_DATA_DIR="$MINT_ROOT_DIR/data" + +declare -A data_file_map +data_file_map["datafile-0-b"]="0" +data_file_map["datafile-1-b"]="1" +data_file_map["datafile-1-kB"]="1K" +data_file_map["datafile-10-kB"]="10K" +data_file_map["datafile-33-kB"]="33K" +data_file_map["datafile-100-kB"]="100K" +data_file_map["datafile-1.03-MB"]="1056K" +data_file_map["datafile-1-MB"]="1M" +data_file_map["datafile-5-MB"]="5M" +data_file_map["datafile-5243880-b"]="5243880" +data_file_map["datafile-6-MB"]="6M" +data_file_map["datafile-10-MB"]="10M" +data_file_map["datafile-11-MB"]="11M" +data_file_map["datafile-65-MB"]="65M" +data_file_map["datafile-129-MB"]="129M" + +mkdir -p "$MINT_DATA_DIR" +for filename in "${!data_file_map[@]}"; do + echo "creating $MINT_DATA_DIR/$filename" + if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then + echo "unable to create data file $MINT_DATA_DIR/$filename" + exit 1 + fi +done diff --git a/mint/entrypoint.sh b/mint/entrypoint.sh new file mode 100755 index 000000000..5e15f6cb7 --- /dev/null +++ b/mint/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +./mint.sh "$@" & + +# Get the pid to be used for kill command if required +main_pid="$!" +trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM +# use -n here to catch mint.sh exit code, notify to ci +wait -n diff --git a/mint/install-packages.list b/mint/install-packages.list new file mode 100644 index 000000000..bc89dbbf7 --- /dev/null +++ b/mint/install-packages.list @@ -0,0 +1,17 @@ +git +python3-pip +nodejs +ruby +ruby-dev +ruby-bundler +php +php7.0-curl +php-xml +default-jre +default-jdk +ant +dirmngr +dotnet-sdk-2.1 +ca-certificates-mono +nuget +libunwind8 diff --git a/mint/mint.sh b/mint/mint.sh new file mode 100755 index 000000000..953c3f096 --- /dev/null +++ b/mint/mint.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# +# Mint (C) 2017, 2018 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONTAINER_ID=$(awk -F / '{ print substr($NF, 1, 12) }' /proc/1/cpuset) +MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} +MINT_MODE=${MINT_MODE:-core} +SERVER_REGION=${SERVER_REGION:-us-east-1} +ENABLE_HTTPS=${ENABLE_HTTPS:-0} +ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} +GO111MODULE=on + +if [ -z "$SERVER_ENDPOINT" ]; then + SERVER_ENDPOINT="play.minio.io:9000" + ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" + SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + ENABLE_HTTPS=1 +fi + +if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then + SERVER_IP="${SERVER_ENDPOINT%%:*}" + SERVER_PORT="${SERVER_ENDPOINT/*:/}" + # Check if SERVER_IP is actually IPv4 address + octets=("${SERVER_IP//./ }") + if [ "${#octets[@]}" -ne 4 ]; then + echo "$SERVER_IP must be an IP address" + exit 1 + fi + for octet in "${octets[@]}"; do + if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then + echo "$SERVER_IP must be an IP address" + exit 1 + fi + done +fi + +ROOT_DIR="$PWD" +TESTS_DIR="$ROOT_DIR/run/core" + +BASE_LOG_DIR="$ROOT_DIR/log" +LOG_FILE="log.json" +ERROR_FILE="error.log" +mkdir -p "$BASE_LOG_DIR" + +function humanize_time() +{ + time="$1" + days=$(( time / 60 / 60 / 24 )) + hours=$(( time / 60 / 60 % 24 )) + minutes=$(( time / 60 % 60 )) + seconds=$(( time % 60 )) + + (( days > 0 )) && echo -n "$days days " + (( hours > 0 )) && echo -n "$hours hours " + (( minutes > 0 )) && echo -n "$minutes minutes " + (( days > 0 || hours > 0 || minutes > 0 )) && echo -n "and " + echo "$seconds seconds" +} + +function run_test() +{ + if [ ! -d "$1" ]; then + return 1 + fi + + start=$(date +%s) + + mkdir -p "$BASE_LOG_DIR/$sdk_name" + + (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE") + rv=$? + end=$(date +%s) + duration=$(humanize_time $(( end - start ))) + + if [ "$rv" -eq 0 ]; then + echo "done in $duration" + else + echo "FAILED in $duration" + entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE") + status=$(jq -e -r .status <<<"$entry") + jq_rv=$? + if [ "$jq_rv" -ne 0 ]; then + echo "$entry" + fi + ## Show error.log when status is empty or not "FAIL". + ## This may happen when test run failed without providing logs. + if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || ([ "$status" != "FAIL" ] && [ "$status" != "fail" ]); then + cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE" + else + jq . <<<"$entry" + fi + fi + return $rv +} + +function trust_s3_endpoint_tls_cert() +{ + # Download the public certificate from the server + openssl s_client -showcerts -connect "$SERVER_ENDPOINT" /dev/null | \ + openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \ + exit -1 + + # Load the certificate in the system + update-ca-certificates --fresh >/dev/null + + # Ask different SDKs/tools to load system certificates + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +} + + +function main() +{ + export MINT_DATA_DIR + export MINT_MODE + export SERVER_ENDPOINT + export SERVER_IP + export SERVER_PORT + + export ACCESS_KEY + export SECRET_KEY + export ENABLE_HTTPS + export SERVER_REGION + export ENABLE_VIRTUAL_STYLE + export GO111MODULE + + echo "Running with" + echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" + echo "ACCESS_KEY: $ACCESS_KEY" + echo "SECRET_KEY: ***REDACTED***" + echo "ENABLE_HTTPS: $ENABLE_HTTPS" + echo "SERVER_REGION: $SERVER_REGION" + echo "MINT_DATA_DIR: $MINT_DATA_DIR" + echo "MINT_MODE: $MINT_MODE" + echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" + echo + echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" + echo + + [ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert + + declare -a run_list + if [ "$MINT_MODE" == "worm" ]; then + if [ "$#" -gt 1 ]; then + echo "No argument is accepted for worm mode" + exit 1 + fi + + run_list=( "$TESTS_DIR/worm" ) + else + sdks=( "$@" ) + + ## populate all sdks except worm when no argument is given. + if [ "$#" -eq 0 ]; then + sdks=( $(ls -I worm "$TESTS_DIR") ) + fi + + for sdk in "${sdks[@]}"; do + if [ "$sdk" == "worm" ]; then + echo "worm test cannot be run without worm mode" + exit 1 + fi + + run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" ) + done + fi + + count="${#run_list[@]}" + i=0 + for sdk_dir in "${run_list[@]}"; do + sdk_name=$(basename "$sdk_dir") + (( i++ )) + if [ ! -d "$sdk_dir" ]; then + echo "Test $sdk_name not found. Exiting Mint." + exit 1 + fi + echo -n "($i/$count) Running $sdk_name tests ... " + if ! run_test "$sdk_dir"; then + (( i-- )) + break + fi + done + + ## Report when all tests in run_list are run + if [ $i -eq "$count" ]; then + echo -e "\nAll tests ran successfully" + else + echo -e "\nExecuted $i out of $count tests successfully." + exit 1 + fi +} +main "$@" diff --git a/mint/postinstall.sh b/mint/postinstall.sh new file mode 100755 index 000000000..ff424807f --- /dev/null +++ b/mint/postinstall.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# remove all packages listed in remove-packages.list +xargs --arg-file=remove-packages.list apt --quiet --yes purge +apt --quiet --yes autoremove + +# remove unwanted files +rm -fr "$GOROOT" "$GOPATH/src" /var/lib/apt/lists/* + +# flush to disk +sync diff --git a/mint/preinstall.sh b/mint/preinstall.sh new file mode 100755 index 000000000..554843a3e --- /dev/null +++ b/mint/preinstall.sh @@ -0,0 +1,49 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export APT="apt --quiet --yes" +export WGET="wget --quiet --no-check-certificate" + +# install nodejs source list +if ! $WGET --output-document=- https://deb.nodesource.com/setup_6.x | bash -; then + echo "unable to set nodejs repository" + exit 1 +fi + +$APT install apt-transport-https + +wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +dpkg -i packages-microsoft-prod.deb +rm -f packages-microsoft-prod.deb +$APT install apt-transport-https +$APT update + +# download and install golang +GO_VERSION="1.12.5" +GO_INSTALL_PATH="/usr/local" +download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" +if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then + echo "unable to install go$GO_VERSION" + exit 1 +fi + +xargs --arg-file=install-packages.list apt --quiet --yes install + +# set python 3.5 as default +update-alternatives --install /usr/bin/python python /usr/bin/python3.5 1 + +sync diff --git a/mint/release.sh b/mint/release.sh new file mode 100755 index 000000000..d302455b3 --- /dev/null +++ b/mint/release.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e +# +# Minio Cloud Storage, (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} +export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" +export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security" +export WGET="wget --quiet --no-check-certificate" + +./create-data-files.sh +./preinstall.sh + +# install mint app packages +for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do + echo "Running $pkg" + $pkg +done + +./postinstall.sh diff --git a/mint/remove-packages.list b/mint/remove-packages.list new file mode 100644 index 000000000..a6c9ab148 --- /dev/null +++ b/mint/remove-packages.list @@ -0,0 +1,9 @@ +wget +git +python3-pip +ruby-dev +ruby-bundler +default-jdk +ant +dotnet +nuget diff --git a/mint/run/core/aws-sdk-go/quick-tests.go b/mint/run/core/aws-sdk-go/quick-tests.go new file mode 100644 index 000000000..b09e243cb --- /dev/null +++ b/mint/run/core/aws-sdk-go/quick-tests.go @@ -0,0 +1,328 @@ +/* +* +* Mint, (C) 2017 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "math/rand" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +func cleanup(s3Client *s3.S3, bucket string, object string, function string, + args map[string]interface{}, startTime time.Time, deleteBucket bool) { + + // Deleting the object, just in case it was created. Will not check for errors. + s3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + }) + + if deleteBucket { + _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go DeleteBucket Failed", err).Fatal() + return + } + } + +} + +func testPresignedPutInvalidHash(s3Client *s3.S3) { + startTime := time.Now() + function := "PresignedPut" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") + object := "presignedTest" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() + return + } + defer cleanup(s3Client, bucket, object, function, args, startTime, true) + + req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + ContentType: aws.String("application/octet-stream"), + }) + + req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256") + url, err := req.Presign(expiry) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal() + return + } + + rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte(""))) + rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256") + rreq.Header.Add("Content-Type", "application/octet-stream") + + resp, err := http.DefaultClient.Do(rreq) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal() + return + } + defer resp.Body.Close() + + dec := xml.NewDecoder(resp.Body) + errResp := ErrorResponse{} + err = dec.Decode(&errResp) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal() + return + } + + if errResp.Code != "XAmzContentSHA256Mismatch" { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +func testListObjects(s3Client *s3.S3) { + startTime := time.Now() + function := "testListObjects" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") + object1 := "testObject1" + object2 := "testObject2" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName1": object1, + "objectName2": object2, + "expiry": expiry, + } + + getKeys := func(objects []*s3.Object) []string { + var rv []string + for _, obj := range objects { + rv = append(rv, *obj.Key) + } + return rv + } + _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() + return + } + defer cleanup(s3Client, bucket, object1, function, args, startTime, true) + defer cleanup(s3Client, bucket, object2, function, args, startTime, false) + + listInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(1000), + Prefix: aws.String(""), + } + result, err := s3Client.ListObjectsV2(listInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() + return + } + if *result.KeyCount != 0 { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 0 key but got %v, %v", result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() + return + } + putInput1 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), + Bucket: aws.String(bucket), + Key: aws.String(object1), + } + _, err = s3Client.PutObject(putInput1) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() + return + } + putInput2 := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), + Bucket: aws.String(bucket), + Key: aws.String(object2), + } + _, err = s3Client.PutObject(putInput2) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() + return + } + result, err = s3Client.ListObjectsV2(listInput) + if err != nil { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() + return + } + if *result.KeyCount != 2 { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 2 key but got %v, %v", *result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() + return + } + + successLogger(function, args, startTime).Info() +} + +func main() { + endpoint := os.Getenv("SERVER_ENDPOINT") + accessKey := os.Getenv("ACCESS_KEY") + secretKey := os.Getenv("SECRET_KEY") + secure := os.Getenv("ENABLE_HTTPS") + sdkEndpoint := "http://" + endpoint + if secure == "1" { + sdkEndpoint = "https://" + endpoint + } + + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + newSession := session.New() + s3Config := &aws.Config{ + Credentials: creds, + Endpoint: aws.String(sdkEndpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + } + + // Create an S3 service object in the default region. + s3Client := s3.New(newSession, s3Config) + + // Output to stdout instead of the default stderr + log.SetOutput(os.Stdout) + // create custom formatter + mintFormatter := mintJSONFormatter{} + // set custom formatter + log.SetFormatter(&mintFormatter) + // log Info or above -- success cases are Info level, failures are Fatal level + log.SetLevel(log.InfoLevel) + // execute tests + testPresignedPutInvalidHash(s3Client) + testListObjects(s3Client) +} diff --git a/mint/run/core/aws-sdk-go/run.sh b/mint/run/core/aws-sdk-go/run.sh new file mode 100755 index 000000000..1811bdd53 --- /dev/null +++ b/mint/run/core/aws-sdk-go/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-java/run.sh b/mint/run/core/aws-sdk-java/run.sh new file mode 100755 index 000000000..30c1e5c68 --- /dev/null +++ b/mint/run/core/aws-sdk-java/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Mint (C) 2018 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +cd /mint/run/core/aws-sdk-java/ || exit -1 + +java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-php/README.md b/mint/run/core/aws-sdk-php/README.md new file mode 100644 index 000000000..d08162d08 --- /dev/null +++ b/mint/run/core/aws-sdk-php/README.md @@ -0,0 +1,19 @@ +## `aws-sdk-php` tests +This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added into `quick-tests.php` as new functions. + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/aws-sdk-php/composer.json b/mint/run/core/aws-sdk-php/composer.json new file mode 100644 index 000000000..9d74e1750 --- /dev/null +++ b/mint/run/core/aws-sdk-php/composer.json @@ -0,0 +1,6 @@ +{ + "require": { + "aws/aws-sdk-php": "^3.30", + "GuzzleHttp/Psr7": "^1.4" + } +} diff --git a/mint/run/core/aws-sdk-php/quick-tests.php b/mint/run/core/aws-sdk-php/quick-tests.php new file mode 100644 index 000000000..754d6dcd6 --- /dev/null +++ b/mint/run/core/aws-sdk-php/quick-tests.php @@ -0,0 +1,1101 @@ + 'val-1']; + +/** + * ClientConfig abstracts configuration details to connect to a + * S3-like service + */ +class ClientConfig { + public $creds; + public $endpoint; + public $region; + + function __construct(string $access_key, string $secret_key, string $host, string $secure, string $region) { + $this->creds = new Aws\Credentials\Credentials($access_key, $secret_key); + + if ($secure == "1") { + $this->endpoint = "https://" . $host; + } else { + $this->endpoint = "http://" . $host; + } + + $this->region = $region; + } +} + + /** + * randomName returns a name prefixed by aws-sdk-php using uniqid() + * from standard library + * + * @return string + */ +function randomName():string { + return uniqid("aws-sdk-php-"); +} + + /** + * getStatusCode returns HTTP status code of the given result. + * + * @param $result - AWS\S3 result object + * + * @return string - HTTP status code. E.g, "400" for Bad Request. + */ +function getStatusCode($result):string { + return $result->toArray()['@metadata']['statusCode']; +} + + /** + * runExceptionalTests executes a collection of tests that will throw + * a known exception. + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $apiCall Name of the S3Client API method to call + * + * @param $exceptionMatcher Name of Aws\S3\Exception\S3Exception + * method to fetch exception details + * + * @param $exceptionParamMap Associative array of exception names to + * API parameters. E.g, + * $apiCall = 'headBucket' + * $exceptionMatcher = 'getStatusCode' + * $exceptionParamMap = [ + * // Non existent bucket + * '404' => ['Bucket' => $bucket['Name'] . '--'], + * + * // Non existent bucket + * '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], + * ]; + * + * @return string - HTTP status code. E.g, "404" for Non existent bucket. + */ +function runExceptionalTests($s3Client, $apiCall, $exceptionMatcher, $exceptionParamMap) { + foreach($exceptionParamMap as $exn => $params) { + $exceptionCaught = false; + try { + $result = $s3Client->$apiCall($params); + } catch(Aws\S3\Exception\S3Exception $e) { + $exceptionCaught = true; + switch ($e->$exceptionMatcher()) { + case $exn: + // This is expected + continue; + default: + throw $e; + } + } + finally { + if (!$exceptionCaught) { + $message = sprintf("Expected %s to fail with %s", $apiCall, $exn); + throw new Exception($message); + } + } + } +} + + /** + * testListBuckets tests ListBuckets S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @return void + */ +function testListBuckets(S3Client $s3Client) { + $buckets = $s3Client->listBuckets(); + $debugger = $GLOBALS['debugger']; + foreach ($buckets['Buckets'] as $bucket){ + $debugger->out($bucket['Name'] . "\n"); + } +} + + /** + * testBucketExists tests HEAD Bucket S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @return void + */ +function testBucketExists(S3Client $s3Client) { + // List all buckets + $buckets = $s3Client->listBuckets(); + // All HEAD on existing buckets must return success + foreach($buckets['Buckets'] as $bucket) { + $result = $s3Client->headBucket(['Bucket' => $bucket['Name']]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('headBucket API failed for ' . $bucket['Name']); + } + + // Run failure tests + $params = [ + // Non existent bucket + '404' => ['Bucket' => $bucket['Name'] . '--'], + + // Non existent bucket + '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], + ]; + runExceptionalTests($s3Client, 'headBucket', 'getStatusCode', $params); +} + + + /** + * testHeadObject tests HeadObject S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $objects Associative array of buckets and objects + * + * @return void + */ +function testHeadObject($s3Client, $objects) { + foreach($objects as $bucket => $object) { + $result = $s3Client->headObject(['Bucket' => $bucket, 'Key' => $object]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('headObject API failed for ' . + $bucket . '/' . $object); + if ($result['Metadata'] != TEST_METADATA) { + throw new Exception("headObject API Metadata didn't match for " . + $bucket . '/' . $object); + } + } + + // Run failure tests + $params = [ + '404' => ['Bucket' => $bucket, 'Key' => $object . '-non-existent'] + ]; + runExceptionalTests($s3Client, 'headObject', 'getStatusCode', $params); +} + + /** + * testListObjects tests ListObjectsV1 and V2 S3 APIs + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testListObjects($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + $debugger = $GLOBALS['debugger']; + try { + for ($i = 0; $i < 5; $i++) { + $copyKey = $object . '-copy-' . strval($i); + $result = $s3Client->copyObject([ + 'Bucket' => $bucket, + 'Key' => $copyKey, + 'CopySource' => $bucket . '/' . $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception("copyObject API failed for " . $bucket . '/' . $object); + } + + $paginator = $s3Client->getPaginator('ListObjects', ['Bucket' => $bucket]); + foreach ($paginator->search('Contents[].Key') as $key) { + $debugger->out('key = ' . $key . "\n"); + } + + $paginator = $s3Client->getPaginator('ListObjectsV2', ['Bucket' => $bucket]); + foreach ($paginator->search('Contents[].Key') as $key) { + $debugger->out('key = ' . $key . "\n"); + } + + $prefix = 'obj'; + $result = $s3Client->listObjects(['Bucket' => $bucket, 'Prefix' => $prefix]); + if (getStatusCode($result) != HTTP_OK || $result['Prefix'] != $prefix) + throw new Exception("listObject API failed for " . $bucket . '/' . $object); + + $maxKeys = 1; + $result = $s3Client->listObjects(['Bucket' => $bucket, 'MaxKeys' => $maxKeys]); + if (getStatusCode($result) != HTTP_OK || count($result['Contents']) != $maxKeys) + throw new Exception("listObject API failed for " . $bucket . '/' . $object); + + $params = [ + 'InvalidArgument' => ['Bucket' => $bucket, 'MaxKeys' => -1], + 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'] + ]; + runExceptionalTests($s3Client, 'listObjects', 'getAwsErrorCode', $params); + + } finally { + $s3Client->deleteObjects([ + 'Bucket' => $bucket, + 'Delete' => [ + 'Objects' => array_map(function($a, $b) { + return ['Key' => $a . '-copy-' . strval($b)]; + }, array_fill(0, 5, $object), range(0,4)) + ], + ]); + } +} + + /** + * testListMultipartUploads tests ListMultipartUploads, ListParts and + * UploadPartCopy S3 APIs + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testListMultipartUploads($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + $debugger = $GLOBALS['debugger']; + + $data_dir = $GLOBALS['MINT_DATA_DIR']; + // Initiate multipart upload + $result = $s3Client->createMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('createMultipartupload API failed for ' . + $bucket . '/' . $object); + + // upload 5 parts + $uploadId = $result['UploadId']; + $parts = []; + try { + for ($i = 0; $i < 5; $i++) { + $result = $s3Client->uploadPartCopy([ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'UploadId' => $uploadId, + 'PartNumber' => $i+1, + 'CopySource' => $bucket . '/' . $object, + ]); + if (getStatusCode($result) != HTTP_OK) { + throw new Exception('uploadPart API failed for ' . + $bucket . '/' . $object); + } + array_push($parts, [ + 'ETag' => $result['ETag'], + 'PartNumber' => $i+1, + ]); + } + + // ListMultipartUploads and ListParts may return empty + // responses in the case of minio gateway gcs and minio server + // FS mode. So, the following tests don't make assumptions on + // result response. + $paginator = $s3Client->getPaginator('ListMultipartUploads', + ['Bucket' => $bucket]); + foreach ($paginator->search('Uploads[].{Key: Key, UploadId: UploadId}') as $keyHash) { + $debugger->out('key = ' . $keyHash['Key'] . ' uploadId = ' . $keyHash['UploadId'] . "\n"); + } + + $paginator = $s3Client->getPaginator('ListParts', [ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'UploadId' => $uploadId, + ]); + foreach ($paginator->search('Parts[].{PartNumber: PartNumber, ETag: ETag}') as $partsHash) { + $debugger->out('partNumber = ' . $partsHash['PartNumber'] . ' ETag = ' . $partsHash['ETag'] . "\n"); + } + + }finally { + $s3Client->abortMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'UploadId' => $uploadId + ]); + } +} + + /** + * initSetup creates buckets and objects necessary for the functional + * tests to run + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $objects Associative array of buckets and objects + * + * @return void + */ +function initSetup(S3Client $s3Client, $objects) { + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + foreach($objects as $bucket => $object) { + $s3Client->createBucket(['Bucket' => $bucket]); + $stream = NULL; + try { + if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) + throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); + + $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); + $result = $s3Client->putObject([ + 'Bucket' => $bucket, + 'Key' => $object, + 'Body' => $stream, + 'Metadata' => TEST_METADATA, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception("putObject API failed for " . $bucket . '/' . $object); + } + + finally { + // close data file + if (!is_null($stream)) + $stream->close(); + } + } + + // Create an empty bucket for bucket policy + delete tests + $result = $s3Client->createBucket(['Bucket' => $GLOBALS['emptyBucket']]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception("createBucket API failed for " . $bucket); + +} + + + /** + * testGetPutObject tests GET/PUT object S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testGetPutObject($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + // Upload a 10KB file + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + try { + $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); + $result = $s3Client->putObject([ + 'Bucket' => $bucket, + 'Key' => $object, + 'Body' => $stream, + ]); + } + finally { + $stream->close(); + } + + if (getStatusCode($result) != HTTP_OK) + throw new Exception("putObject API failed for " . $bucket . '/' . $object); + + // Download the same object and verify size + $result = $s3Client->getObject([ + 'Bucket' => $bucket, + 'Key' => $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception("getObject API failed for " . $bucket . '/' . $object); + + $body = $result['Body']; + $bodyLen = 0; + while (!$body->eof()) { + $bodyLen += strlen($body->read(4096)); + } + + if ($bodyLen != 1 * 1024) { + throw new Exception("Object downloaded has different content length than uploaded object " + . $bucket . '/' . $object); + } +} + + /** + * testMultipartUploadFailure tests MultipartUpload failures + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testMultipartUploadFailure($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + // Initiate multipart upload + $result = $s3Client->createMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('createMultipartupload API failed for ' . + $bucket . '/' . $object); + + // upload 2 parts + $uploadId = $result['UploadId']; + $parts = []; + try { + for ($i = 0; $i < 2; $i++) { + $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); + $limitedStream = new Psr7\LimitStream($stream, 4 * 1024 * 1024, 0); + $result = $s3Client->uploadPart([ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => $uploadId, + 'ContentLength' => 4 * 1024 * 1024, + 'Body' => $limitedStream, + 'PartNumber' => $i+1, + ]); + if (getStatusCode($result) != HTTP_OK) { + throw new Exception('uploadPart API failed for ' . + $bucket . '/' . $object); + } + array_push($parts, [ + 'ETag' => $result['ETag'], + 'PartNumber' => $i+1, + ]); + + $limitedStream->close(); + $limitedStream = NULL; + } + } + finally { + if (!is_null($limitedStream)) + $limitedStream->close(); + } + + $params = [ + 'EntityTooSmall' => [ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => $uploadId, + 'MultipartUpload' => [ + 'Parts' => $parts, + ], + ], + 'NoSuchUpload' => [ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => 'non-existent', + 'MultipartUpload' => [ + 'Parts' => $parts, + ], + ], + ]; + runExceptionalTests($s3Client, 'completeMultipartUpload', 'getAwsErrorCode', $params); +} + + /** + * testMultipartUpload tests MultipartUpload S3 APIs + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testMultipartUpload($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + // Initiate multipart upload + $result = $s3Client->createMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('createMultipartupload API failed for ' . + $bucket . '/' . $object); + + // upload 2 parts + $uploadId = $result['UploadId']; + $parts = []; + try { + for ($i = 0; $i < 2; $i++) { + $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); + $result = $s3Client->uploadPart([ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => $uploadId, + 'ContentLength' => 5 * 1024 * 1024, + 'Body' => $stream, + 'PartNumber' => $i+1, + ]); + if (getStatusCode($result) != HTTP_OK) { + throw new Exception('uploadPart API failed for ' . + $bucket . '/' . $object); + } + array_push($parts, [ + 'ETag' => $result['ETag'], + 'PartNumber' => $i+1, + ]); + + $stream->close(); + $stream = NULL; + } + } + finally { + if (!is_null($stream)) + $stream->close(); + } + + // complete multipart upload + $result = $s3Client->completeMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => $uploadId, + 'MultipartUpload' => [ + 'Parts' => $parts, + ], + ]); + if (getStatusCode($result) != HTTP_OK) { + throw new Exception('completeMultipartupload API failed for ' . + $bucket . '/' . $object); + } +} + + /** + * testAbortMultipartUpload tests aborting of a multipart upload + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testAbortMultipartUpload($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + // Initiate multipart upload + $result = $s3Client->createMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('createMultipartupload API failed for ' . + $bucket . '/' . $object); + + // Abort multipart upload + $uploadId = $result['UploadId']; + $result = $s3Client->abortMultipartUpload([ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => $uploadId, + ]); + if (getStatusCode($result) != HTTP_NOCONTENT) + throw new Exception('abortMultipartupload API failed for ' . + $bucket . '/' . $object); + + //Run failure tests + $params = [ + // Upload doesn't exist + 'NoSuchUpload' => [ + 'Bucket' => $bucket, + 'Key' => $object, + 'UploadId' => 'non-existent', + ], + ]; + runExceptionalTests($s3Client, 'abortMultipartUpload', 'getAwsErrorCode', $params); +} + + /** + * testGetBucketLocation tests GET bucket location S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket name + * + * @return void + */ +function testGetBucketLocation($s3Client, $params) { + $bucket = $params['Bucket']; + + // Valid test + $result = $s3Client->getBucketLocation(['Bucket' => $bucket]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('getBucketLocation API failed for ' . + $bucket); + + // Run failure tests. + $params = [ + // Non existent bucket + 'NoSuchBucket' => ['Bucket' => $bucket . '--'], + + // Bucket not found + 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'], + ]; + runExceptionalTests($s3Client, 'getBucketLocation', 'getAwsErrorCode', $params); +} + + /** + * testCopyObject tests copy object S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object name + * + * @return void + */ +function testCopyObject($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + $result = $s3Client->copyObject([ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'CopySource' => $bucket . '/' . $object, + ]); + if (getStatusCode($result) != HTTP_OK) + throw new Exception('copyObject API failed for ' . + $bucket); + + $s3Client->deleteObject([ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + ]); + + // Run failure tests + $params = [ + // Invalid copy source format + 'InvalidArgument' => [ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'CopySource' => $bucket . $object + ], + + // Missing source object + 'NoSuchKey' => [ + 'Bucket' => $bucket, + 'Key' => $object . '-copy', + 'CopySource' => $bucket . '/' . $object . '-non-existent' + ], + ]; + runExceptionalTests($s3Client, 'copyObject', 'getAwsErrorCode', $params); +} + + /** + * testDeleteObjects tests Delete Objects S3 API + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testDeleteObjects($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + $copies = []; + for ($i = 0; $i < 3; $i++) { + $copyKey = $object . '-copy' . strval($i); + $result = $s3Client->copyObject([ + 'Bucket' => $bucket, + 'Key' => $copyKey, + 'CopySource' => $bucket . '/' . $object, + ]); + if (getstatuscode($result) != HTTP_OK) + throw new Exception('copyobject API failed for ' . + $bucket); + array_push($copies, ['Key' => $copyKey]); + } + + $result = $s3Client->deleteObjects([ + 'Bucket' => $bucket, + 'Delete' => [ + 'Objects' => $copies, + ], + ]); + if (getstatuscode($result) != HTTP_OK) + throw new Exception('deleteObjects api failed for ' . + $bucket); +} + + /** + * testAnonDeleteObjects tests Delete Objects S3 API for anonymous requests. + * The test case checks this scenario: + * http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html#multiobjectdeleteapi-examples + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testAnonDeleteObjects($s3Client, $params) { + $bucket = $params['Bucket']; + $object = $params['Object']; + + // Create anonymous config object + $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); + + // Create anonymous S3 client + $anonymousClient = new S3Client([ + 'credentials' => false, + 'endpoint' => $anonConfig->endpoint, + 'use_path_style_endpoint' => true, + 'region' => $anonConfig->region, + 'version' => '2006-03-01' + ]); + + $copies = []; + for ($i = 0; $i < 3; $i++) { + $copyKey = $object . '-copy' . strval($i); + $result = $s3Client->copyObject([ + 'Bucket' => $bucket, + 'Key' => $copyKey, + 'CopySource' => $bucket . '/' . $object, + ]); + if (getstatuscode($result) != HTTP_OK) + throw new Exception('copyobject API failed for ' . + $bucket); + array_push($copies, ['Key' => $copyKey]); + } + + // Try anonymous delete. + $result = $anonymousClient->deleteObjects([ + 'Bucket' => $bucket, + 'Delete' => [ + 'Objects' => $copies, + ], + ]); + // Response code should be 200 + if (getstatuscode($result) != HTTP_OK) + throw new Exception('deleteObjects returned incorrect response ' . + getStatusCode($result)); + + // Each object should have error code AccessDenied + for ($i = 0; $i < 3; $i++) { + if ($result["Errors"][$i]["Code"] != "AccessDenied") + throw new Exception('Incorrect response deleteObjects anonymous + call for ' .$bucket); + } + + // Delete objects after the test passed + $result = $s3Client->deleteObjects([ + 'Bucket' => $bucket, + 'Delete' => [ + 'Objects' => $copies, + ], + ]); + + if (getstatuscode($result) != HTTP_OK) + throw new Exception('deleteObjects api failed for ' . + $bucket); + + // Each object should have empty code in case of successful delete + for ($i = 0; $i < 3; $i++) { + if ($result["Errors"][$i]["Code"] != "") + throw new Exception('Incorrect response deleteObjects anonymous + call for ' .$bucket); + } +} + + /** + * testBucketPolicy tests GET/PUT Bucket policy S3 APIs + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $params associative array containing bucket and object names + * + * @return void + */ +function testBucketPolicy($s3Client, $params) { + $bucket = $params['Bucket']; + + // Taken from policy set using `mc policy download` + $downloadPolicy = sprintf('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}', $bucket); + + $result = $s3Client->putBucketPolicy([ + 'Bucket' => $bucket, + 'Policy' => $downloadPolicy + ]); + if (getstatuscode($result) != HTTP_NOCONTENT) + throw new Exception('putBucketPolicy API failed for ' . + $bucket); + $result = $s3Client->getBucketPolicy(['Bucket' => $bucket]); + if (getstatuscode($result) != HTTP_OK) + throw new Exception('putBucketPolicy API failed for ' . + $bucket); + + if ($result['Policy'] != $downloadPolicy) + throw new Exception('bucket policy we got is not we set'); + + // Delete the bucket, make the bucket (again) and check if policy is none + // Ref: https://github.com/minio/minio/issues/4714 + $result = $s3Client->deleteBucket(['Bucket' => $bucket]); + if (getstatuscode($result) != HTTP_NOCONTENT) + throw new Exception('deleteBucket API failed for ' . + $bucket); + + try { + $s3Client->getBucketPolicy(['Bucket' => $bucket]); + } catch (AWSException $e) { + switch ($e->getAwsErrorCode()) { + case 'NoSuchBucket': + break; + } + } + + // Sleep is needed for Minio Gateway for Azure, ref: + // https://docs.microsoft.com/en-us/rest/api/storageservices/Delete-Container#remarks + sleep(40); + + $s3Client->createBucket(['Bucket' => $bucket]); + + $params = [ + '404' => ['Bucket' => $bucket] + ]; + runExceptionalTests($s3Client, 'getBucketPolicy', 'getStatusCode', $params); + + try { + $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; + // Create an object to test anonymous GET object + $object = 'test-anon'; + if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) + throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); + + $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); + $result = $s3Client->putObject([ + 'Bucket' => $bucket, + 'Key' => $object, + 'Body' => $stream, + ]); + if (getstatuscode($result) != HTTP_OK) + throw new Exception('createBucket API failed for ' . + $bucket); + + $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); + $anonymousClient = new S3Client([ + 'credentials' => false, + 'endpoint' => $anonConfig->endpoint, + 'use_path_style_endpoint' => true, + 'region' => $anonConfig->region, + 'version' => '2006-03-01' + ]); + runExceptionalTests($anonymousClient, 'getObject', 'getStatusCode', [ + '403' => [ + 'Bucket' => $bucket, + 'Key' => $object, + ] + ]); + + } finally { + // close data file + if (!is_null($stream)) + $stream->close(); + $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); + } + +} + + /** + * cleanupSetup removes all buckets and objects created during the + * functional test + * + * @param $s3Client AWS\S3\S3Client object + * + * @param $objects Associative array of buckets to objects + * + * @return void + */ +function cleanupSetup($s3Client, $objects) { + // Delete all objects + foreach ($objects as $bucket => $object) { + $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); + } + + // Delete the buckets incl. emptyBucket + $allBuckets = array_keys($objects); + array_push($allBuckets, $GLOBALS['emptyBucket']); + foreach ($allBuckets as $bucket) { + try { + // Delete the bucket + $s3Client->deleteBucket(['Bucket' => $bucket]); + + // Wait until the bucket is removed from object store + $s3Client->waitUntil('BucketNotExists', ['Bucket' => $bucket]); + } catch (Exception $e) { + // Ignore exceptions thrown during cleanup + } + } +} + + + /** + * runTest helper function to wrap a test function and log + * success or failure accordingly. + * + * @param myfunc name of test function to be run + * + * @param fnSignature function signature of the main S3 SDK API + * + * @param args parameters to be passed to test function + * + * @return void + */ +function runTest($s3Client, $myfunc, $fnSignature, $args = []) { + try { + $start_time = microtime(true); + $status = "PASS"; + $error = ""; + $message = ""; + $myfunc($s3Client, $args); + } catch (AwsException $e) { + $errorCode = $e->getAwsErrorCode(); + // $fnSignature holds the specific API that is being + // tested. It is possible that functions used to create the + // test setup may not be implemented. + if ($errorCode != "NotImplemented") { + $status = "FAIL"; + $error = $e->getMessage(); + } else { + $status = "NA"; + $error = $e->getMessage(); + $alert = sprintf("%s or a related API is NOT IMPLEMENTED, see \"error\" for exact details.", $fnSignature); + } + + } catch (Exception $e) { + // This exception handler handles high-level custom exceptions. + $status = "FAIL"; + $error = $e->getMessage(); + } finally { + $end_time = microtime(true); + $json_log = [ + "name" => "aws-sdk-php", + "function" => $fnSignature, + "args" => $args, + "duration" => sprintf("%d", ($end_time - $start_time) * 1000), // elapsed time in ms + "status" => $status, + ]; + if ($error !== "") { + $json_log["error"] = $error; + } + if ($message !== "") { + $json_log["message"] = $message; + } + print_r(json_encode($json_log)."\n"); + + // Exit on first failure. + switch ($status) { + case "FAIL": + exit(1); + } + } +} + +// Get client configuration from environment variables +$GLOBALS['access_key'] = getenv("ACCESS_KEY"); +$GLOBALS['secret_key'] = getenv("SECRET_KEY"); +$GLOBALS['endpoint'] = getenv("SERVER_ENDPOINT"); +$GLOBALS['region'] = getenv("SERVER_REGION"); +$GLOBALS['secure'] = getenv("ENABLE_HTTPS"); + +/** + * @global string $GLOBALS['MINT_DATA_DIR'] + * @name $MINT_DATA_DIR + */ +$GLOBALS['MINT_DATA_DIR'] = '/mint/data'; +$GLOBALS['MINT_DATA_DIR'] = getenv("MINT_DATA_DIR"); + + +// Useful for debugging test failures; Set $debugmode it to true when required +$debugmode = false; + +interface Debugger { + public function out($data); +} + +class EchoDebugger implements Debugger { + public function out($data) { + echo $data; + } +} + +class NullDebugger implements Debugger { + public function out($data) { + // Do nothing + } +} + +if($debugmode) + $debugger = new EchoDebugger(); +else + $debugger = new NullDebugger(); + +// Make $debugger global +$GLOBALS['debugger'] = $debugger; + +// Create config object +$config = new ClientConfig($GLOBALS['access_key'], $GLOBALS['secret_key'], + $GLOBALS['endpoint'], $GLOBALS['secure'], + $GLOBALS['region']); + +// Create a S3Client +$s3Client = new S3Client([ + 'credentials' => $config->creds, + 'endpoint' => $config->endpoint, + 'use_path_style_endpoint' => true, + 'region' => $config->region, + 'version' => '2006-03-01' +]); + +// Used by initSetup +$emptyBucket = randomName(); +$objects = [ + randomName() => 'obj1', + randomName() => 'obj2', +]; + +try { + initSetup($s3Client, $objects); + $firstBucket = array_keys($objects)[0]; + $firstObject = $objects[$firstBucket]; + $testParams = ['Bucket' => $firstBucket, 'Object' => $firstObject]; + runTest($s3Client, 'testGetBucketLocation', "getBucketLocation ( array \$params = [] )", ['Bucket' => $firstBucket]); + runTest($s3Client, 'testListBuckets', "listBuckets ( array \$params = [] )"); + runTest($s3Client, 'testListObjects', "listObjects ( array \$params = [] )", $testParams); + runTest($s3Client, 'testListMultipartUploads', "listMultipartUploads ( array \$params = [] )", $testParams); + runTest($s3Client, 'testBucketExists', "headBucket ( array \$params = [] )", array_keys($objects)); + runTest($s3Client, 'testHeadObject', "headObject ( array \$params = [] )", $objects); + runTest($s3Client, 'testGetPutObject', "getObject ( array \$params = [] )", $testParams); + runTest($s3Client, 'testCopyObject', "copyObject ( array \$params = [] )", $testParams); + runTest($s3Client, 'testDeleteObjects', "deleteObjects (array \$params = [] )", $testParams); + runTest($s3Client, 'testAnonDeleteObjects', "anonDeleteObjects ( array \$params = [] )", $testParams); + runTest($s3Client, 'testMultipartUpload', "createMultipartUpload ( array \$params = [] )", $testParams); + runTest($s3Client, 'testMultipartUploadFailure', "uploadPart ( array \$params = [] )", $testParams); + runTest($s3Client, 'testAbortMultipartUpload', "abortMultipartupload ( array \$params = [] )", $testParams); + runTest($s3Client, 'testBucketPolicy', "getBucketPolicy ( array \$params = [] )", ['Bucket' => $emptyBucket]); +} +finally { + cleanupSetup($s3Client, $objects); +} + +?> diff --git a/mint/run/core/aws-sdk-php/run.sh b/mint/run/core/aws-sdk-php/run.sh new file mode 100755 index 000000000..3a5c0b890 --- /dev/null +++ b/mint/run/core/aws-sdk-php/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint, (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-ruby/README.md b/mint/run/core/aws-sdk-ruby/README.md new file mode 100644 index 000000000..7c66adae4 --- /dev/null +++ b/mint/run/core/aws-sdk-ruby/README.md @@ -0,0 +1,19 @@ +## `aws-sdk-ruby` tests +This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added into `aws-stub-test.rb` as new functions. + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb b/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb new file mode 100755 index 000000000..185004ded --- /dev/null +++ b/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb @@ -0,0 +1,855 @@ +#!/usr/bin/env ruby +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'aws-sdk' +require 'securerandom' +require 'net/http' +require 'multipart_body' + +class AwsSdkRubyTest + # Set variables necessary to create an s3 client instance. + # Get them from the environment variables + + # Region information, eg. "us-east-1" + region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set' + # Minio server, eg. "play.minio.io:9000" + endpoint = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set' + access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set' + secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set' + enable_https = ENV['ENABLE_HTTPS'] + endpoint = enable_https == '1' ? 'https://' + endpoint : 'http://' + endpoint + # Create s3 client instances, "s3Resource" and "s3Client" + @@s3 = Aws::S3::Resource.new(region: region, + endpoint: endpoint, + access_key_id: access_key_id, + secret_access_key: secret_access_key, + force_path_style: true) + + def initialize_log_output(meth, alert = nil) + # Initialize and return log content in log_output hash table + + # Collect args in args_arr + args_arr = method(meth).parameters.flatten.map(&:to_s) + .reject { |x| x == 'req' || x == 'opt' } + # Create and return log output content + { name: 'aws-sdk-ruby', + function: "#{meth}(#{args_arr.join(',')})", # method name and arguments + args: args_arr, # array of arg names. This'll be replaced with a + # a arg/value pairs insdie the caller method + duration: 0, # test runtime duration in seconds + alert: alert, + message: nil, + error: nil } + end + + def get_random_bucket_name() + bucket_name = "aws-sdk-ruby-bucket-"+SecureRandom.hex(6) + return bucket_name + end + + def calculate_duration(t2, t1) + # Durations are in miliseconds, with precision of 2 decimal places + ((t2 - t1) * 1000).round(2) + end + + def print_log(log_output, start_time) + # Calculate duration in miliseconds + log_output[:duration] = calculate_duration(Time.now, start_time) + # Get rid of the log_output fields if nil + puts log_output.delete_if{|k, value| value == nil}.to_json + # Exit at the first failure + exit 1 if log_output[:status] == 'FAIL' + end + + def cleanUp(buckets, log_output) + # Removes objects and bucket if bucket exists + bucket_name = '' + buckets.each do |b| + bucket_name = b + if bucketExistsWrapper(b, log_output) + removeObjectsWrapper(b, log_output) + removeBucketWrapper(b, log_output) + end + end + rescue => e + raise "Failed to clean-up bucket '#{bucket_name}', #{e}" + end + + # + # API commands/methods + # + def makeBucket(bucket_name) + # Creates a bucket, "bucket_name" + # on S3 client , "s3". + # Returns bucket_name if already exists + @@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name) + rescue => e + raise e + end + + def makeBucketWrapper(bucket_name, log_output) + makeBucket(bucket_name) + rescue => e + log_output[:function] = "makeBucket(bucket_name)" + log_output[:args] = {'bucket_name': bucket_name} + raise e + end + + def removeBucket(bucket_name) + # Deletes/removes bucket, "bucket_name" on S3 client, "s3" + @@s3.bucket(bucket_name).delete + rescue => e + raise e + end + + def removeBucketWrapper(bucket_name, log_output) + removeBucket(bucket_name) + rescue => e + log_output[:function] = "removeBucket(bucket_name)" + log_output[:args] = {'bucket_name': bucket_name} + raise e + end + + def putObject(bucket_name, file) + # Creates "file" (full path) in bucket, "bucket_name", + # on S3 client, "s3" + file_name = File.basename(file) + @@s3.bucket(bucket_name).object(file_name).upload_file(file) + rescue => e + raise e + end + + def putObjectWrapper(bucket_name, file, log_output) + putObject(bucket_name, file) + rescue => e + log_output[:function] = "putObject(bucket_name, file)" + log_output[:args] = {'bucket_name': bucket_name, + 'file': file} + raise e + end + + def getObject(bucket_name, file, destination) + # Gets/Downloads file, "file", + # from bucket, "bucket_name", of S3 client, "s3" + file_name = File.basename(file) + dest = File.join(destination, file_name) + @@s3.bucket(bucket_name).object(file_name).get(response_target: dest) + rescue => e + raise e + end + + def getObjectWrapper(bucket_name, file, destination, log_output) + getObject(bucket_name, file, destination) + rescue => e + log_output[:function] = "getObject(bucket_name, file)" + log_output[:args] = {'bucket_name': bucket_name, + 'file': file, + 'destination': destination} + raise e + end + + def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '') + # Copies file, "file_name", from source bucket, + # "source_bucket_name", to target bucket, + # "target_bucket_name", on S3 client, "s3" + target_file_name = source_file_name if target_file_name.empty? + source = @@s3.bucket(source_bucket_name) + target = @@s3.bucket(target_bucket_name) + source_obj = source.object(source_file_name) + target_obj = target.object(target_file_name) + source_obj.copy_to(target_obj) + rescue => e + raise e + end + + def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output) + copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name) + rescue => e + log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')' + log_output[:args] = {'source_bucket_name': source_bucket_name, + 'target_bucket_name': target_bucket_name, + 'source_file_name': source_file_name, + 'target_file_name': target_file_name} + raise e + end + + def removeObject(bucket_name, file) + # Deletes file in bucket, + # "bucket_name", on S3 client, "s3". + # If file, "file_name" does not exist, + # it quietly returns without any error message + @@s3.bucket(bucket_name).object(file).delete + rescue => e + raise e + end + + def removeObjectWrapper(bucket_name, file_name, log_output) + removeObject(bucket_name, file_name) + rescue => e + log_output[:function] = "removeObject(bucket_name, file_name)" + log_output[:args] = {'bucket_name': bucket_name, + 'file_name': file_name} + raise e + end + + def removeObjects(bucket_name) + # Deletes all files in bucket, "bucket_name" + # on S3 client, "s3" + file_name = '' + @@s3.bucket(bucket_name).objects.each do |obj| + file_name = obj.key + obj.delete + end + rescue => e + raise "File name: '#{file_name}', #{e}" + end + + def removeObjectsWrapper(bucket_name, log_output) + removeObjects(bucket_name) + rescue => e + log_output[:function] = 'removeObjects(bucket_name)' + log_output[:args] = {'bucket_name': bucket_name} + raise e + end + + def listBuckets + # Returns an array of bucket names on S3 client, "s3" + bucket_name_list = [] + @@s3.buckets.each do |b| + bucket_name_list.push(b.name) + end + return bucket_name_list + rescue => e + raise e + end + + def listBucketsWrapper(log_output) + listBuckets + rescue => e + log_output[:function] = 'listBuckets' + log_output[:args] = {} + raise e + end + + def listObjects(bucket_name) + # Returns an array of object/file names + # in bucket, "bucket_name", on S3 client, "s3" + object_list = [] + @@s3.bucket(bucket_name).objects.each do |obj| + object_list.push(obj.key) + end + return object_list + rescue => e + raise e + end + + def listObjectsWrapper(bucket_name, log_output) + listObjects(bucket_name) + rescue => e + log_output[:function] = 'listObjects(bucket_name)' + log_output[:args] = {'bucket_name': bucket_name} + raise e + end + + def statObject(bucket_name, file_name) + return @@s3.bucket(bucket_name).object(file_name).exists? + rescue => e + raise e + end + + def statObjectWrapper(bucket_name, file_name, log_output) + statObject(bucket_name, file_name) + rescue => e + log_output[:function] = 'statObject(bucket_name, file_name)' + log_output[:args] = {'bucket_name': bucket_name, + 'file_name': file_name} + raise e + end + + def bucketExists?(bucket_name) + # Returns true if bucket, "bucket_name", exists, + # false otherwise + return @@s3.bucket(bucket_name).exists? + rescue => e + raise e + end + + def bucketExistsWrapper(bucket_name, log_output) + bucketExists?(bucket_name) + rescue => e + log_output[:function] = 'bucketExists?(bucket_name)' + log_output[:args] = {'bucket_name': bucket_name} + raise e + end + + def presignedGet(bucket_name, file_name) + # Returns download/get url + obj = @@s3.bucket(bucket_name).object(file_name) + return obj.presigned_url(:get, expires_in: 600) + rescue => e + raise e + end + + def presignedGetWrapper(bucket_name, file_name, log_output) + presignedGet(bucket_name, file_name) + rescue => e + log_output[:function] = 'presignedGet(bucket_name, file_name)' + log_output[:args] = {'bucket_name': bucket_name, + 'file_name': file_name} + raise e + end + + def presignedPut(bucket_name, file_name) + # Returns put url + obj = @@s3.bucket(bucket_name).object(file_name) + return obj.presigned_url(:put, expires_in: 600) + rescue => e + raise e + end + + def presignedPutWrapper(bucket_name, file_name, log_output) + presignedPut(bucket_name, file_name) + rescue => e + log_output[:function] = 'presignedPut(bucket_name, file_name)' + log_output[:args] = {'bucket_name': bucket_name, + 'file_name': file_name} + raise e + end + + def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) + # Returns upload/post url + obj = @@s3.bucket(bucket_name).object(file_name) + return obj.presigned_post(expires: Time.now + expires_in_sec, + content_length_range: 1..max_byte_size) + rescue => e + raise e + end + + def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output) + presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) + rescue => e + log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)' + log_output[:args] = {'bucket_name': bucket_name, + 'file_name': file_name, + 'expires_in_sec': expires_in_sec, + 'max_byte_size': max_byte_size} + raise e + end + + # To be addressed. S3 API 'get_bucket_policy' does not work! + # def getBucketPolicy(bucket_name) + # # Returns bucket policy + # return @@s3.bucket(bucket_name).get_bucket_policy + # rescue => e + # raise e + # end + + # + # Test case methods + # + def listBucketsTest() + # Tests listBuckets api command by creating + # new buckets from bucket_name_list + + # get random bucket names and create list + bucket_name1 = get_random_bucket_name() + bucket_name2 = get_random_bucket_name() + bucket_name_list = [bucket_name1, bucket_name2] + # Initialize hash table, 'log_output' + log_output = initialize_log_output('listBuckets') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + prev_total_buckets = listBucketsWrapper(log_output).length + new_buckets = bucket_name_list.length + bucket_name_list.each do |b| + makeBucketWrapper(b, log_output) + end + new_total_buckets = prev_total_buckets + new_buckets + if new_total_buckets >= prev_total_buckets + new_buckets + log_output[:status] = 'PASS' + else + log_output[:error] = 'Could not find expected number of buckets' + log_output[:status] = 'FAIL' + end + cleanUp(bucket_name_list, log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def makeBucketTest() + # Tests makeBucket api command. + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('makeBucket') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + + if bucketExistsWrapper(bucket_name, log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = 'Bucket expected to be created does not exist' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def bucketExistsNegativeTest() + # Tests bucketExists api command. + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('bucketExists?') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + if !bucketExistsWrapper(bucket_name, log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = "Failed to return 'false' for a non-existing bucket" + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def removeBucketTest() + # Tests removeBucket api command. + + # get a random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('removeBucket') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + removeBucketWrapper(bucket_name, log_output) + if !bucketExistsWrapper(bucket_name, log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = 'Bucket expected to be removed still exists' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def putObjectTest(file) + # Tests putObject api command by uploading a file + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('putObject') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + putObjectWrapper(bucket_name, file, log_output) + if statObjectWrapper(bucket_name, File.basename(file), log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = "Status for the created object returned 'false'" + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def removeObjectTest(file) + # Tests removeObject api command by uploading and removing a file + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('removeObject') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + putObjectWrapper(bucket_name, file, log_output) + removeObjectWrapper(bucket_name, File.basename(file), log_output) + if !statObjectWrapper(bucket_name, File.basename(file), log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = "Status for the removed object returned 'true'" + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def getObjectTest(file, destination) + # Tests getObject api command + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('getObject') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + putObjectWrapper(bucket_name, file, log_output) + getObjectWrapper(bucket_name, file, destination, log_output) + if system("ls -l #{destination} > /dev/null") + log_output[:status] = 'PASS' + else + log_output[:error] = "Downloaded object does not exist at #{destination}" + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def listObjectsTest(file_list) + # Tests listObjects api command + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('listObjects') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + # Put all objects into the bucket + file_list.each do |f| + putObjectWrapper(bucket_name, f, log_output) + end + # Total number of files uploaded + expected_no = file_list.length + # Actual number is what api returns + actual_no = listObjectsWrapper(bucket_name, log_output).length + # Compare expected and actual values + if expected_no == actual_no + log_output[:status] = 'PASS' + else + log_output[:error] = 'Expected and actual number of listed files/objects do not match!' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def copyObjectTest(data_dir, source_file_name, target_file_name = '') + # Tests copyObject api command + + # get random bucket names + source_bucket_name = get_random_bucket_name() + target_bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('copyObject') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + target_file_name = source_file_name if target_file_name.empty? + makeBucketWrapper(source_bucket_name, log_output) + makeBucketWrapper(target_bucket_name, log_output) + putObjectWrapper(source_bucket_name, + File.join(data_dir, source_file_name), log_output) + copyObjectWrapper(source_bucket_name, target_bucket_name, + source_file_name, target_file_name, log_output) + # Check if copy worked fine + if statObjectWrapper(target_bucket_name, target_file_name, log_output) + log_output[:status] = 'PASS' + else + log_output[:error] = 'Copied file could not be found in the expected location' + log_output[:status] = 'FAIL' + end + cleanUp([source_bucket_name, target_bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def presignedGetObjectTest(data_dir, file_name) + # Tests presignedGetObject api command + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('presignedGet') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + file = File.join(data_dir, file_name) + # Get check sum value without the file name + cksum_orig = `cksum #{file}`.split[0..1] + putObjectWrapper(bucket_name, file, log_output) + get_url = presignedGetWrapper(bucket_name, file_name, log_output) + # Download the file using the URL + # generated by presignedGet api command + `wget -O /tmp/#{file_name}, '#{get_url}' > /dev/null 2>&1` + # Get check sum value for the downloaded file + # Split to get rid of the file name + cksum_new = `cksum /tmp/#{file_name}`.split[0..1] + + # Check if check sum values for the orig file + # and the downloaded file match + if cksum_orig == cksum_new + log_output[:status] = 'PASS' + else + log_output[:error] = 'Check sum values do NOT match' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def presignedPutObjectTest(data_dir, file_name) + # Tests presignedPutObject api command + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('presignedPut') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + file = File.join(data_dir, file_name) + + # Get check sum value and + # split to get rid of the file name + cksum_orig = `cksum #{file}`.split[0..1] + + # Generate presigned Put URL and parse it + uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output)) + request = Net::HTTP::Put.new(uri.request_uri, 'x-amz-acl' => 'public-read') + request.body = IO.read(File.join(data_dir, file_name)) + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' + + http.request(request) + + if statObjectWrapper(bucket_name, file_name, log_output) + getObjectWrapper(bucket_name, file_name, '/tmp', log_output) + cksum_new = `cksum /tmp/#{file_name}`.split[0..1] + # Check if check sum values of the orig file + # and the downloaded file match + if cksum_orig == cksum_new + log_output[:status] = 'PASS' + else + log_output[:error] = 'Check sum values do NOT match' + log_output[:status] = 'FAIL' + end + else + log_output[:error] = 'Expected to be created object does NOT exist' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end + + def presignedPostObjectTest(data_dir, file_name, + expires_in_sec, max_byte_size) + # Tests presignedPostObject api command + + # get random bucket name + bucket_name = get_random_bucket_name() + # Initialize hash table, 'log_output' + log_output = initialize_log_output('presignedPost') + # Prepare arg/value hash table and set it in log_output + arg_value_hash = {} + log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } + log_output[:args] = arg_value_hash + + begin + start_time = Time.now + makeBucketWrapper(bucket_name, log_output) + file = File.join(data_dir, file_name) + + # Get check sum value and split it + # into parts to get rid of the file name + cksum_orig = `cksum #{file}`.split[0..1] + # Create the presigned POST url + post = presignedPostWrapper(bucket_name, file_name, + expires_in_sec, max_byte_size, log_output) + + # Prepare multi parts array for POST command request + file_part = Part.new name: 'file', + body: IO.read(File.join(data_dir, file_name)), + filename: file_name, + content_type: 'application/octet-stream' + parts = [file_part] + # Add POST fields into parts array + post.fields.each do |field, value| + parts.push(Part.new(field, value)) + end + boundary = "---------------------------#{rand(10_000_000_000_000_000)}" + body_parts = MultipartBody.new parts, boundary + + # Parse presigned Post URL + uri = URI.parse(post.url) + + # Create the HTTP objects + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' + request = Net::HTTP::Post.new(uri.request_uri) + request.body = body_parts.to_s + request.content_type = "multipart/form-data; boundary=#{boundary}" + # Send the request + log_output[:error] = http.request(request) + + if statObjectWrapper(bucket_name, file_name, log_output) + getObjectWrapper(bucket_name, file_name, '/tmp', log_output) + cksum_new = `cksum /tmp/#{file_name}`.split[0..1] + # Check if check sum values of the orig file + # and the downloaded file match + if cksum_orig == cksum_new + log_output[:status] = 'PASS' + # FIXME: HTTP No Content error, status code=204 is returned as error + log_output[:error] = nil + else + log_output[:error] = 'Check sum values do NOT match' + log_output[:status] = 'FAIL' + end + else + log_output[:error] = 'Expected to be created object does NOT exist' + log_output[:status] = 'FAIL' + end + cleanUp([bucket_name], log_output) + rescue => log_output[:error] + log_output[:status] = 'FAIL' + end + + print_log(log_output, start_time) + end +end + +# MAIN CODE + +# Create test Class instance and call the tests +aws = AwsSdkRubyTest.new +file_name1 = 'datafile-1-kB' +file_new_name = 'datafile-1-kB-copy' +file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB'] +# Add data_dir in front of each file name in file_name_list +# The location where the bucket and file +# objects are going to be created. +data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set' +file_list = file_name_list.map { |f| File.join(data_dir, f) } +destination = '/tmp' + +aws.listBucketsTest() +aws.listObjectsTest(file_list) +aws.makeBucketTest() +aws.bucketExistsNegativeTest() +aws.removeBucketTest() +aws.putObjectTest(File.join(data_dir, file_name1)) +aws.removeObjectTest(File.join(data_dir, file_name1)) +aws.getObjectTest(File.join(data_dir, file_name1), destination) +aws.copyObjectTest(data_dir, file_name1) +aws.copyObjectTest(data_dir, file_name1, file_new_name) +aws.presignedGetObjectTest(data_dir, file_name1) +aws.presignedPutObjectTest(data_dir, file_name1) +aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024) diff --git a/mint/run/core/aws-sdk-ruby/run.sh b/mint/run/core/aws-sdk-ruby/run.sh new file mode 100755 index 000000000..73e647b79 --- /dev/null +++ b/mint/run/core/aws-sdk-ruby/run.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +chmod a+x aws-stub-tests.rb +ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/README.md b/mint/run/core/awscli/README.md new file mode 100644 index 000000000..b395f9a7c --- /dev/null +++ b/mint/run/core/awscli/README.md @@ -0,0 +1,19 @@ +## `awscli` tests +This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added into `test.sh` as new functions. + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/awscli/run.sh b/mint/run/core/awscli/run.sh new file mode 100755 index 000000000..7ce73ee17 --- /dev/null +++ b/mint/run/core/awscli/run.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# configure awscli +aws configure set aws_access_key_id "$ACCESS_KEY" +aws configure set aws_secret_access_key "$SECRET_KEY" +aws configure set default.region "$SERVER_REGION" + +# run tests for virtual style if provided +if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then + # Setup endpoint scheme + endpoint="http://$DOMAIN:$SERVER_PORT" + if [ "$ENABLE_HTTPS" -eq 1 ]; then + endpoint="https://$DOMAIN:$SERVER_PORT" + fi + dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root + echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf + aws configure set default.s3.addressing_style virtual + ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" + aws configure set default.s3.addressing_style path +fi + +endpoint="http://$SERVER_ENDPOINT" +if [ "$ENABLE_HTTPS" -eq 1 ]; then + endpoint="https://$SERVER_ENDPOINT" +fi +# run path style tests +./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/test.sh b/mint/run/core/awscli/test.sh new file mode 100755 index 000000000..46c82b0b7 --- /dev/null +++ b/mint/run/core/awscli/test.sh @@ -0,0 +1,1411 @@ +#!/bin/bash +# +# Mint (C) 2017, 2018 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +HASH_1_KB=$(md5sum "${MINT_DATA_DIR}/datafile-1-kB" | awk '{print $1}') +HASH_65_MB=$(md5sum "${MINT_DATA_DIR}/datafile-65-MB" | awk '{print $1}') + +_init() { + AWS="aws --endpoint-url $1" +} + +function get_time() { + date +%s%N +} + +function get_duration() { + start_time=$1 + end_time=$(get_time) + + echo $(( (end_time - start_time) / 1000000 )) +} + +function log_success() { + function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") + printf '{"name": "awscli", "duration": %d, "function": %s, "status": "PASS"}\n' "$1" "$function" +} + +function log_failure() { + function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") + err=$(echo "$3" | tr -d '\n') + printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "error": "%s"}\n' "$1" "$function" "$err" +} + +function log_alert() { + function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") + err=$(echo "$4" | tr -d '\n') + printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "alert": "%s", "error": "%s"}\n' "$1" "$function" "$3" "$err" +} + +function make_bucket() { + # Make bucket + bucket_name="awscli-mint-test-bucket-$RANDOM" + function="${AWS} s3api create-bucket --bucket ${bucket_name}" + + # execute the test + out=$($function 2>&1) + rv=$? + + # if command is successful print bucket_name or print error + if [ $rv -eq 0 ]; then + echo "${bucket_name}" + else + echo "${out}" + fi + + return $rv +} + +function delete_bucket() { + # Delete bucket + function="${AWS} s3 rb s3://${1} --force" + out=$($function 2>&1) + rv=$? + + # echo the output + echo "${out}" + + return $rv +} + +# Tests creating, stat and delete on a bucket. +function test_create_bucket() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + # save the ref to function being tested, so it can be logged + test_function=${function} + + # if make_bucket is successful stat the bucket + if [ $rv -eq 0 ]; then + function="${AWS} s3api head-bucket --bucket ${bucket_name}" + out=$($function 2>&1) + rv=$? + else + # if make bucket failes, $bucket_name has the error output + out="${bucket_name}" + fi + + # if stat bucket is successful remove the bucket + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "${bucket_name}") + rv=$? + else + # if make bucket failes, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests creating and deleting an object. +function test_upload_object() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + # if upload succeeds download the file + if [ $rv -eq 0 ]; then + function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB /tmp/datafile-1-kB" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + # calculate the md5 hash of downloaded file + hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') + fi + + # if download succeeds, verify downloaded file + if [ $rv -eq 0 ]; then + if [ "$HASH_1_KB" == "$hash2" ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove download file + rm -f /tmp/datafile-1-kB + else + rv=1 + out="Checksum verification failed for uploaded object" + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Test lookup a directory prefix. +function test_lookup_object_prefix() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds create a directory. + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" + # save the ref to function being tested, so it can be logged + test_function=${function} + + out=$($function 2>&1) + + rv=$? + else + # if make_bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + ## Attempt an overwrite of the prefix again and should succeed as well. + function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # if upload succeeds lookup for the prefix. + if [ $rv -eq 0 ]; then + function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # if directory create succeeds, upload the object. + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it. + if [ $rv -eq 0 ]; then + function="${AWS} s3api delete-object --bucket ${bucket_name} --key prefix/directory/" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # if upload succeeds lookup for the object should succeed. + if [ $rv -eq 0 ]; then + function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" + # save the ref to function being tested, so it can be logged + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # delete bucket + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + fi + + if [ $rv -ne 0 ]; then + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + else + log_success "$(get_duration "$start_time")" "${test_function}" + fi + + return $rv +} + +# Tests listing objects for both v1 and v2 API. +function test_list_objects() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + # if upload objects succeeds, list objects with existing prefix + if [ $rv -eq 0 ]; then + function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB" + test_function=${function} + out=$($function) + rv=$? + key_name=$(echo "$out" | jq -r .Contents[].Key) + if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then + rv=1 + # since rv is 0, command passed, but didn't return expected value. In this case set the output + out="list-objects with existing prefix failed" + fi + fi + + # if upload objects succeeds, list objects without existing prefix + if [ $rv -eq 0 ]; then + function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix linux" + out=$($function) + rv=$? + key_name=$(echo "$out" | jq -r .Contents[].Key) + if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then + rv=1 + out="list-objects without existing prefix failed" + fi + fi + + # if upload objects succeeds, list objectsv2 with existing prefix + if [ $rv -eq 0 ]; then + function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB" + out=$($function) + rv=$? + key_name=$(echo "$out" | jq -r .Contents[].Key) + if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then + rv=1 + out="list-objects-v2 with existing prefix failed" + fi + fi + + # if upload objects succeeds, list objectsv2 without existing prefix + if [ $rv -eq 0 ]; then + function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix linux" + out=$($function) + rv=$? + key_name=$(echo "$out" | jq -r .Contents[].Key) + if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then + rv=1 + out="list-objects-v2 without existing prefix failed" + fi + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove download file + rm -f /tmp/datafile-1-kB + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + rm -f /tmp/datafile-1-kB + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests multipart API with 0 byte part. +function test_multipart_upload_0byte() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + object_name=${bucket_name}"-object" + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-0-b --bucket ${bucket_name} --key datafile-0-b" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + # create multipart + function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" + test_function=${function} + out=$($function) + rv=$? + upload_id=$(echo "$out" | jq -r .UploadId) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 1 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-0-b --upload-id ${upload_id} --part-number 1" + out=$($function) + rv=$? + etag1=$(echo "$out" | jq -r .ETag) + fi + + if [ $rv -eq 0 ]; then + # Create a multipart struct file for completing multipart transaction + echo "{ + \"Parts\": [ + { + \"ETag\": ${etag1}, + \"PartNumber\": 1 + } + ] + }" >> /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + # Use saved etags to complete the multipart transaction + function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" + out=$($function) + rv=$? + etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') + if [ "${etag}" == "" ]; then + rv=1 + out="complete-multipart-upload failed" + fi + fi + + if [ $rv -eq 0 ]; then + function="${AWS} s3api get-object --bucket ${bucket_name} --key ${object_name} /tmp/datafile-0-b" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + if [ $rv -eq 0 ]; then + ret_etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') + # match etag + if [ "$etag" != "$ret_etag" ]; then + rv=1 + out="Etag mismatch for multipart 0 byte object" + fi + rm -f /tmp/datafile-0-b + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove temp file + rm -f /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + rm -f /tmp/multipart + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests multipart API by making each individual calls. +function test_multipart_upload() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + object_name=${bucket_name}"-object" + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + # create multipart + function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" + test_function=${function} + out=$($function) + rv=$? + upload_id=$(echo "$out" | jq -r .UploadId) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 1 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5-MB --upload-id ${upload_id} --part-number 1" + out=$($function) + rv=$? + etag1=$(echo "$out" | jq -r .ETag) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 2 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-1-kB --upload-id ${upload_id} --part-number 2" + out=$($function) + rv=$? + etag2=$(echo "$out" | jq -r .ETag) + # Create a multipart struct file for completing multipart transaction + echo "{ + \"Parts\": [ + { + \"ETag\": ${etag1}, + \"PartNumber\": 1 + }, + { + \"ETag\": ${etag2}, + \"PartNumber\": 2 + } + ] + }" >> /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + # Use saved etags to complete the multipart transaction + function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" + out=$($function) + rv=$? + finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') + if [ "${finalETag}" == "" ]; then + rv=1 + out="complete-multipart-upload failed" + fi + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove temp file + rm -f /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + rm -f /tmp/multipart + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# List number of objects based on the maxKey +# value set. +function test_max_key_list() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-b --bucket ${bucket_name} --key datafile-1-b" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + # copy object server side + if [ $rv -eq 0 ]; then + function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-b-copy --copy-source ${bucket_name}/datafile-1-b" + out=$($function) + rv=$? + fi + + if [ $rv -eq 0 ]; then + function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --max-keys 1" + test_function=${function} + out=$($function 2>&1) + rv=$? + if [ $rv -eq 0 ]; then + out=$(echo "$out" | jq '.KeyCount') + rv=$? + fi + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # The command passed, but the delete_bucket failed + out="delete_bucket for test_max_key_list failed" + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Copy object tests for server side copy +# of the object, validates returned md5sum. +function test_copy_object() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + # copy object server side + if [ $rv -eq 0 ]; then + function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB" + test_function=${function} + out=$($function) + rv=$? + hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') + if [ $rv -eq 0 ] && [ "$HASH_1_KB" == "$hash2" ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # The command passed, but the verification failed + out="Verification failed for copied object" + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests for presigned URL success case, presigned URL +# is correct and accessible - we calculate md5sum of +# the object and validate it against a local files md5sum. +function test_presigned_object() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + function="${AWS} s3 presign s3://${bucket_name}/datafile-1-kB" + test_function=${function} + url=$($function) + rv=$? + curl -sS -X GET "${url}" > /tmp/datafile-1-kB + hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') + if [ "$HASH_1_KB" == "$hash2" ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove download file + rm -f /tmp/datafile-1-kB + else + rv=1 + out="Checksum verification failed for downloaded object" + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests creating and deleting an object - 10MiB +function test_upload_object_10() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-MB --bucket ${bucket_name} --key datafile-10-MB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests multipart API by making each individual calls with 10MiB part size. +function test_multipart_upload_10() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + object_name=${bucket_name}"-object" + rv=$? + + if [ $rv -eq 0 ]; then + # create multipart + function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" + test_function=${function} + out=$($function) + rv=$? + upload_id=$(echo "$out" | jq -r .UploadId) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 1 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 1" + out=$($function) + rv=$? + etag1=$(echo "$out" | jq -r .ETag) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 2 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 2" + out=$($function) + rv=$? + etag2=$(echo "$out" | jq -r .ETag) + # Create a multipart struct file for completing multipart transaction + echo "{ + \"Parts\": [ + { + \"ETag\": ${etag1}, + \"PartNumber\": 1 + }, + { + \"ETag\": ${etag2}, + \"PartNumber\": 2 + } + ] + }" >> /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + # Use saved etags to complete the multipart transaction + function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" + out=$($function) + rv=$? + finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') + if [ "${finalETag}" == "" ]; then + rv=1 + out="complete-multipart-upload failed" + fi + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove temp file + rm -f /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + rm -f /tmp/multipart + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests `aws s3 cp` by uploading a local file. +function test_aws_s3_cp() { + file_name="${MINT_DATA_DIR}/datafile-65-MB" + + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file using cp + if [ $rv -eq 0 ]; then + function="${AWS} s3 cp $file_name s3://${bucket_name}/$(basename "$file_name")" + test_function=${function} + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + function="${AWS} s3 rm s3://${bucket_name}/$(basename "$file_name")" + out=$($function 2>&1) + rv=$? + fi + + if [ $rv -eq 0 ]; then + function="${AWS} s3 rb s3://${bucket_name}/" + out=$($function 2>&1) + rv=$? + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# Tests `aws s3 sync` by mirroring all the +# local content to remove bucket. +function test_aws_s3_sync() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds sync all the files in a directory + if [ $rv -eq 0 ]; then + function="${AWS} s3 sync $MINT_DATA_DIR s3://${bucket_name}/" + test_function=${function} + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + # remove files recusively + if [ $rv -eq 0 ]; then + function="${AWS} s3 rm --recursive s3://${bucket_name}/" + out=$($function 2>&1) + rv=$? + fi + + # delete bucket + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# list objects negative test - tests for following conditions. +# v1 API with max-keys=-1 and max-keys=0 +# v2 API with max-keys=-1 and max-keys=0 +function test_list_objects_error() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload a file + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" + out=$($function 2>&1) + rv=$? + else + # if make bucket fails, $bucket_name has the error output + out="${bucket_name}" + fi + + if [ $rv -eq 0 ]; then + # Server replies an error for v1 with max-key=-1 + function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" + test_function=${function} + out=$($function 2>&1) + rv=$? + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + fi + + if [ $rv -eq 0 ]; then + # Server replies an error for v2 with max-keys=-1 + function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" + test_function=${function} + out=$($function 2>&1) + rv=$? + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + fi + + if [ $rv -eq 0 ]; then + # Server returns success with no keys when max-keys=0 + function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=0" + out=$($function 2>&1) + rv=$? + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# put object negative test - tests for following conditions. +# - invalid object name. +# - invalid Content-Md5 +# - invalid Content-Length +function test_put_object_error() { + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # if make bucket succeeds upload an object without content-md5. + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-md5 invalid" + test_function=${function} + out=$($function 2>&1) + rv=$? + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + fi + + # upload an object without content-length. + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-length -1" + test_function=${function} + out=$($function 2>&1) + rv=$? + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + fi + + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} +# tests server side encryption headers for get and put calls +function test_serverside_encryption() { + #skip server side encryption tests if HTTPS disabled. + if [ "$ENABLE_HTTPS" != "1" ]; then + return 0 + fi + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # put object with server side encryption headers + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + # now get encrypted object from server + if [ $rv -eq 0 ]; then + etag1=$(echo "$out" | jq -r .ETag) + sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) + sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) + + function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + if [ $rv -eq 0 ]; then + etag2=$(echo "$out" | jq -r .ETag) + sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) + sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) + hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') + # match downloaded object's hash to original + if [ "$HASH_1_KB" == "$hash2" ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove download file + rm -f /tmp/datafile-1-kB + else + rv=1 + out="Checksum verification failed for downloaded object" + fi + # match etag and SSE headers + if [ "$etag1" != "$etag2" ]; then + rv=1 + out="Etag mismatch for object encrypted with server side encryption" + fi + if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then + rv=1 + out="sse customer algorithm mismatch" + fi + if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then + rv=1 + out="sse customer key mismatch" + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# tests server side encryption headers for multipart put +function test_serverside_encryption_multipart() { + #skip server side encryption tests if HTTPS disabled. + if [ "$ENABLE_HTTPS" != "1" ]; then + return 0 + fi + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # put object with server side encryption headers + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-65-MB --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + # now get encrypted object from server + if [ $rv -eq 0 ]; then + etag1=$(echo "$out" | jq -r .ETag) + sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) + sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) + + function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-65-MB" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + if [ $rv -eq 0 ]; then + etag2=$(echo "$out" | jq -r .ETag) + sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) + sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) + hash2=$(md5sum /tmp/datafile-65-MB | awk '{print $1}') + # match downloaded object's hash to original + if [ "$HASH_65_MB" == "$hash2" ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + # remove download file + rm -f /tmp/datafile-65-MB + else + rv=1 + out="Checksum verification failed for downloaded object" + fi + # match etag and SSE headers + if [ "$etag1" != "$etag2" ]; then + rv=1 + out="Etag mismatch for object encrypted with server side encryption" + fi + if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then + rv=1 + out="sse customer algorithm mismatch" + fi + if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then + rv=1 + out="sse customer key mismatch" + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# tests encrypted copy from multipart encrypted object to +# single part encrypted object. This test in particular checks if copy +# succeeds for the case where encryption overhead for individually +# encrypted parts vs encryption overhead for the original datastream +# differs. +function test_serverside_encryption_multipart_copy() { + #skip server side encryption tests if HTTPS disabled. + if [ "$ENABLE_HTTPS" != "1" ]; then + return 0 + fi + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + object_name=${bucket_name}"-object" + rv=$? + + if [ $rv -eq 0 ]; then + # create multipart + function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name} --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + out=$($function) + rv=$? + upload_id=$(echo "$out" | jq -r .UploadId) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 1 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + out=$($function) + rv=$? + etag1=$(echo "$out" | jq -r .ETag) + fi + + if [ $rv -eq 0 ]; then + # Capture etag for part-number 2 + function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + out=$($function) + rv=$? + etag2=$(echo "$out" | jq -r .ETag) + # Create a multipart struct file for completing multipart transaction + echo "{ + \"Parts\": [ + { + \"ETag\": ${etag1}, + \"PartNumber\": 1 + }, + { + \"ETag\": ${etag2}, + \"PartNumber\": 2 + } + ] + }" >> /tmp/multipart + fi + + if [ $rv -eq 0 ]; then + # Use saved etags to complete the multipart transaction + function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" + out=$($function) + rv=$? + finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') + if [ "${finalETag}" == "" ]; then + rv=1 + out="complete-multipart-upload failed" + fi + fi + + # copy object server side + if [ $rv -eq 0 ]; then + function="${AWS} s3api copy-object --bucket ${bucket_name} --key ${object_name}-copy --copy-source ${bucket_name}/${object_name} --copy-source-sse-customer-algorithm AES256 --copy-source-sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --copy-source-sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function) + rv=$? + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + fi + + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + rm -f /tmp/multipart + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} +# tests server side encryption headers for range get calls +function test_serverside_encryption_get_range() { + #skip server side encryption tests if HTTPS disabled. + if [ "$ENABLE_HTTPS" != "1" ]; then + return 0 + fi + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + # put object with server side encryption headers + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-kB --bucket ${bucket_name} --key datafile-10-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + # now get encrypted object from server for range 500-999 + if [ $rv -eq 0 ]; then + etag1=$(echo "$out" | jq -r .ETag) + sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) + sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) + function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-10-kB --range bytes=500-999 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-10-kB" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + if [ $rv -eq 0 ]; then + cnt=$(stat -c%s /tmp/datafile-10-kB) + if [ "$cnt" -ne 500 ]; then + rv=1 + fi + fi + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + return $rv +} + +# tests server side encryption error for get and put calls +function test_serverside_encryption_error() { + #skip server side encryption tests if HTTPS disabled. + if [ "$ENABLE_HTTPS" != "1" ]; then + return 0 + fi + # log start time + start_time=$(get_time) + + function="make_bucket" + bucket_name=$(make_bucket) + rv=$? + + # put object with server side encryption headers with MD5Sum mismatch for sse-customer-key-md5 header + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + # put object with missing server side encryption header sse-customer-algorithm + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + + # put object with server side encryption headers successfully + if [ $rv -eq 0 ]; then + function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + + # now test get on encrypted object with nonmatching sse-customer-key and sse-customer-md5 headers + if [ $rv -eq 0 ]; then + function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" + test_function=${function} + out=$($function 2>&1) + rv=$? + fi + if [ $rv -ne 255 ]; then + rv=1 + else + rv=0 + fi + # delete bucket + if [ $rv -eq 0 ]; then + function="delete_bucket" + out=$(delete_bucket "$bucket_name") + rv=$? + fi + if [ $rv -eq 0 ]; then + log_success "$(get_duration "$start_time")" "${test_function}" + else + # clean up and log error + ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 + log_failure "$(get_duration "$start_time")" "${function}" "${out}" + fi + + return $rv +} + +# main handler for all the tests. +main() { + # Success tests + test_create_bucket && \ + test_upload_object && \ + test_lookup_object_prefix && \ + test_list_objects && \ + test_multipart_upload_0byte && \ + test_multipart_upload && \ + test_max_key_list && \ + test_copy_object && \ + test_presigned_object && \ + test_upload_object_10 && \ + test_multipart_upload_10 && \ + test_serverside_encryption && \ + test_serverside_encryption_get_range && \ + test_serverside_encryption_multipart && \ + test_serverside_encryption_multipart_copy && \ + # Success cli ops. + test_aws_s3_cp && \ + test_aws_s3_sync && \ + # Error tests + test_list_objects_error && \ + test_put_object_error && \ + test_serverside_encryption_error + return $? +} + +_init "$@" && main diff --git a/mint/run/core/healthcheck/healthcheck.go b/mint/run/core/healthcheck/healthcheck.go new file mode 100644 index 000000000..9b25eb15d --- /dev/null +++ b/mint/run/core/healthcheck/healthcheck.go @@ -0,0 +1,195 @@ +/* +* +* Mint, (C) 2019 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "time" + + log "github.com/sirupsen/logrus" +) + +const ( + pass = "PASS" // Indicate that a test passed + fail = "FAIL" // Indicate that a test failed + livenessPath = "/minio/health/live" + readinessPath = "/minio/health/ready" + prometheusPath = "/minio/prometheus/metrics" + timeout = time.Duration(30 * time.Second) +) + +type mintJSONFormatter struct { +} + +func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { + data := make(log.Fields, len(entry.Data)) + for k, v := range entry.Data { + switch v := v.(type) { + case error: + // Otherwise errors are ignored by `encoding/json` + // https://github.com/sirupsen/logrus/issues/137 + data[k] = v.Error() + default: + data[k] = v + } + } + + serialized, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) + } + return append(serialized, '\n'), nil +} + +// log successful test runs +func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { + // calculate the test case duration + duration := time.Since(startTime) + // log with the fields as per mint + fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass} + return log.WithFields(fields) +} + +// log failed test runs +func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { + // calculate the test case duration + duration := time.Since(startTime) + var fields log.Fields + // log with the fields as per mint + if err != nil { + fields = log.Fields{"name": "healthcheck", "function": function, "args": args, + "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err} + } else { + fields = log.Fields{"name": "healthcheck", "function": function, "args": args, + "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message} + } + return log.WithFields(fields) +} + +func testLivenessEndpoint(endpoint string) { + startTime := time.Now() + function := "testLivenessEndpoint" + + u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath)) + if err != nil { + // Could not parse URL successfully + failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal() + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr, Timeout: timeout} + resp, err := client.Get(u.String()) + if err != nil { + // GET request errored + failureLog(function, nil, startTime, "", "GET request failed", err).Fatal() + } + if resp.StatusCode != http.StatusOK { + // Status not 200 OK + failureLog(function, nil, startTime, "", "GET /minio/health/live returned non OK status", err).Fatal() + } + + defer resp.Body.Close() + defer successLogger(function, nil, startTime).Info() +} + +func testReadinessEndpoint(endpoint string) { + startTime := time.Now() + function := "testReadinessEndpoint" + + u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath)) + if err != nil { + // Could not parse URL successfully + failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal() + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr, Timeout: timeout} + resp, err := client.Get(u.String()) + if err != nil { + // GET request errored + failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal() + } + if resp.StatusCode != http.StatusOK { + // Status not 200 OK + failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal() + } + + defer resp.Body.Close() + defer successLogger(function, nil, startTime).Info() +} + +func testPrometheusEndpoint(endpoint string) { + startTime := time.Now() + function := "testPrometheusEndpoint" + + u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, prometheusPath)) + if err != nil { + // Could not parse URL successfully + failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal() + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr, Timeout: timeout} + resp, err := client.Get(u.String()) + if err != nil { + // GET request errored + failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal() + } + if resp.StatusCode != http.StatusOK { + // Status not 200 OK + failureLog(function, nil, startTime, "", "GET /minio/prometheus/metrics returned non OK status", err).Fatal() + } + + defer resp.Body.Close() + defer successLogger(function, nil, startTime).Info() +} + +func main() { + endpoint := os.Getenv("SERVER_ENDPOINT") + secure := os.Getenv("ENABLE_HTTPS") + endpoint = "http://" + endpoint + if secure == "1" { + endpoint = "https://" + endpoint + } + + // Output to stdout instead of the default stderr + log.SetOutput(os.Stdout) + // create custom formatter + mintFormatter := mintJSONFormatter{} + // set custom formatter + log.SetFormatter(&mintFormatter) + // log Info or above -- success cases are Info level, failures are Fatal level + log.SetLevel(log.InfoLevel) + // execute tests + testLivenessEndpoint(endpoint) + testReadinessEndpoint(endpoint) + testPrometheusEndpoint(endpoint) +} diff --git a/mint/run/core/healthcheck/run.sh b/mint/run/core/healthcheck/run.sh new file mode 100755 index 000000000..96e64abe4 --- /dev/null +++ b/mint/run/core/healthcheck/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2019 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/mc/README.md b/mint/run/core/mc/README.md new file mode 100644 index 000000000..9a550e09d --- /dev/null +++ b/mint/run/core/mc/README.md @@ -0,0 +1,19 @@ +## `mc` tests +This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added into `test.sh` as new functions. + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/mc/run.sh b/mint/run/core/mc/run.sh new file mode 100755 index 000000000..2bdc67d4d --- /dev/null +++ b/mint/run/core/mc/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Minio Cloud Storage, (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-dotnet/run.sh b/mint/run/core/minio-dotnet/run.sh new file mode 100755 index 000000000..f5fc00ad8 --- /dev/null +++ b/mint/run/core/minio-dotnet/run.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" +/mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-go/README.md b/mint/run/core/minio-go/README.md new file mode 100644 index 000000000..6f1723b72 --- /dev/null +++ b/mint/run/core/minio-go/README.md @@ -0,0 +1,19 @@ +## `minio-go` tests +This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/minio-go/run.sh b/mint/run/core/minio-go/run.sh new file mode 100755 index 000000000..5ffa4b86f --- /dev/null +++ b/mint/run/core/minio-go/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-java/README.md b/mint/run/core/minio-java/README.md new file mode 100644 index 000000000..8e73d7d48 --- /dev/null +++ b/mint/run/core/minio-java/README.md @@ -0,0 +1,19 @@ +## `minio-java` tests +This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/minio-java/run.sh b/mint/run/core/minio-java/run.sh new file mode 100755 index 000000000..8ed724a4a --- /dev/null +++ b/mint/run/core/minio-java/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +endpoint="http://$SERVER_ENDPOINT" +if [ "$ENABLE_HTTPS" -eq 1 ]; then + endpoint="https://$SERVER_ENDPOINT" +fi + +java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \ + "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-js/README.md b/mint/run/core/minio-js/README.md new file mode 100644 index 000000000..1144a61f4 --- /dev/null +++ b/mint/run/core/minio-js/README.md @@ -0,0 +1,19 @@ +## `minio-js` tests +This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/minio-js/minioreporter.js b/mint/run/core/minio-js/minioreporter.js new file mode 100644 index 000000000..3e77934d0 --- /dev/null +++ b/mint/run/core/minio-js/minioreporter.js @@ -0,0 +1,69 @@ +/* + * Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var mocha = require('mocha'); +module.exports = minioreporter; + +function minioreporter(runner) { + mocha.reporters.Base.call(this, runner); + var self = this; + + runner.on('pass', function (test) { + GenerateJsonEntry(test) + }); + + runner.on('fail', function (test, err) { + GenerateJsonEntry(test, err) + }); + +} + +/** + * Convert test result into a JSON object and print on the console. + * + * @api private + * @param test, err + */ + +function GenerateJsonEntry (test, err) { + var res = test.title.split("_") + var jsonEntry = {}; + + jsonEntry.name = "minio-js" + + if (res.length > 0 && res[0].length) { + jsonEntry.function = res[0] + } + + if (res.length > 1 && res[1].length) { + jsonEntry.args = res[1] + } + + jsonEntry.duration = test.duration + + if (res.length > 2 && res[2].length) { + jsonEntry.alert = res[2] + } + + if (err != null ) { + jsonEntry.status = "FAIL" + jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'') + } else { + jsonEntry.status = "PASS" + } + + process.stdout.write(JSON.stringify(jsonEntry) + "\n") +} diff --git a/mint/run/core/minio-js/package.json b/mint/run/core/minio-js/package.json new file mode 100644 index 000000000..84d581cf2 --- /dev/null +++ b/mint/run/core/minio-js/package.json @@ -0,0 +1,49 @@ +{ + "name": "bin", + "version": "1.0.0", + "main": "functional_test.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "app-module-path": "*", + "async": "*", + "block-stream2": "*", + "concat-stream": "*", + "es6-error": "*", + "json-stream": "*", + "lodash": "*", + "mime-types": "*", + "mkdirp": "*", + "moment": "*", + "source-map-support": "*", + "through2": "*", + "xml": "*", + "xml2js": "*" + }, + "devDependencies": { + "browserify": "*", + "chai": "*", + "gulp": "*", + "gulp-babel": "*", + "gulp-jscs": "*", + "jshint":"2.*", + "gulp-jshint": "*", + "gulp-mocha": "*", + "gulp-notify": "*", + "gulp-sourcemaps": "*", + "jshint-stylish": "*", + "mocha": "*", + "mocha-steps": "*", + "nock": "*", + "rewire": "*", + "superagent": "*" + }, + "scripts": { + "test": "mocha" + } +} diff --git a/mint/run/core/minio-js/run.sh b/mint/run/core/minio-js/run.sh new file mode 100755 index 000000000..99b12b7f3 --- /dev/null +++ b/mint/run/core/minio-js/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Minio Cloud Storage, (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +./node_modules/mocha/bin/mocha -R minioreporter -b --exit 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-py/README.md b/mint/run/core/minio-py/README.md new file mode 100644 index 000000000..36c65cc2f --- /dev/null +++ b/mint/run/core/minio-py/README.md @@ -0,0 +1,19 @@ +## `minio-py` tests +This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/minio-py/run.sh b/mint/run/core/minio-py/run.sh new file mode 100755 index 000000000..7d15357b1 --- /dev/null +++ b/mint/run/core/minio-py/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/.gitignore b/mint/run/core/s3cmd/.gitignore new file mode 100644 index 000000000..518a1c6f3 --- /dev/null +++ b/mint/run/core/s3cmd/.gitignore @@ -0,0 +1,2 @@ +*~ +*.log diff --git a/mint/run/core/s3cmd/README.md b/mint/run/core/s3cmd/README.md new file mode 100644 index 000000000..a961c02c2 --- /dev/null +++ b/mint/run/core/s3cmd/README.md @@ -0,0 +1,19 @@ +## `s3cmd` tests +This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests. + +## Adding new tests +New tests is added into `test.sh` as new functions. + +## Running tests manually +- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` +- Call `run.sh` with output log file and error log file. for example +```bash +export MINT_DATA_DIR=~/my-mint-dir +export MINT_MODE=core +export SERVER_ENDPOINT="play.minio.io:9000" +export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" +export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +export ENABLE_HTTPS=1 +export SERVER_REGION=us-east-1 +./run.sh /tmp/output.log /tmp/error.log +``` diff --git a/mint/run/core/s3cmd/run.sh b/mint/run/core/s3cmd/run.sh new file mode 100755 index 000000000..2d9bca656 --- /dev/null +++ b/mint/run/core/s3cmd/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +./test.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/test.sh b/mint/run/core/s3cmd/test.sh new file mode 100755 index 000000000..e8c415713 --- /dev/null +++ b/mint/run/core/s3cmd/test.sh @@ -0,0 +1,422 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +if [ -n "$MINT_MODE" ]; then + if [ -z "${MINT_DATA_DIR+x}" ]; then + echo "MINT_DATA_DIR not defined" + exit 1 + fi + if [ -z "${SERVER_ENDPOINT+x}" ]; then + echo "SERVER_ENDPOINT not defined" + exit 1 + fi + if [ -z "${ACCESS_KEY+x}" ]; then + echo "ACCESS_KEY not defined" + exit 1 + fi + if [ -z "${SECRET_KEY+x}" ]; then + echo "SECRET_KEY not defined" + exit 1 + fi +fi + +if [ -z "${SERVER_ENDPOINT+x}" ]; then + SERVER_ENDPOINT="play.minio.io:9000" + ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" + SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + ENABLE_HTTPS=1 + SERVER_REGION="us-east-1" +fi + +WORK_DIR="$PWD" +DATA_DIR="$MINT_DATA_DIR" +if [ -z "$MINT_MODE" ]; then + WORK_DIR="$PWD/.run-$RANDOM" + DATA_DIR="$WORK_DIR/data" +fi + +FILE_1_MB="$DATA_DIR/datafile-1-MB" +FILE_65_MB="$DATA_DIR/datafile-65-MB" +declare FILE_1_MB_MD5SUM +declare FILE_65_MB_MD5SUM + +BUCKET_NAME="s3cmd-test-bucket-$RANDOM" +S3CMD=$(which s3cmd) +declare -a S3CMD_CMD + +function get_md5sum() +{ + filename="$FILE_1_MB" + out=$(md5sum "$filename" 2>/dev/null) + rv=$? + if [ "$rv" -eq 0 ]; then + awk '{ print $1 }' <<< "$out" + fi + + return "$rv" +} + +function get_time() +{ + date +%s%N +} + +function get_duration() +{ + start_time=$1 + end_time=$(get_time) + + echo $(( (end_time - start_time) / 1000000 )) +} + +function log_success() +{ + if [ -n "$MINT_MODE" ]; then + printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2" + fi +} + +function show() +{ + if [ -z "$MINT_MODE" ]; then + func_name="$1" + echo "Running $func_name()" + fi +} + +function fail() +{ + rv="$1" + shift + + if [ "$rv" -ne 0 ]; then + echo "$@" + fi + + return "$rv" +} + +function assert() +{ + expected_rv="$1" + shift + start_time="$1" + shift + func_name="$1" + shift + + err=$("$@" 2>&1) + rv=$? + if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then + if [ -n "$MINT_MODE" ]; then + err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))') + ## err is already JSON string, no need to double quote + printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err" + else + echo "s3cmd: $func_name: $err" + fi + + exit "$rv" + fi + + return 0 +} + +function assert_success() { + assert 0 "$@" +} + +function assert_failure() { + assert 1 "$@" +} + +function s3cmd_cmd() +{ + cmd=( "${S3CMD_CMD[@]}" "$@" ) + "${cmd[@]}" + rv=$? + return "$rv" +} + +function check_md5sum() +{ + expected_checksum="$1" + shift + filename="$*" + + checksum="$(get_md5sum "$filename")" + rv=$? + if [ "$rv" -ne 0 ]; then + echo "unable to get md5sum for $filename" + return "$rv" + fi + + if [ "$checksum" != "$expected_checksum" ]; then + echo "$filename: md5sum mismatch" + return 1 + fi + + return 0 +} + +function test_make_bucket() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + bucket_name="s3cmd-test-bucket-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_make_bucket_error() { + show "${FUNCNAME[0]}" + + start_time=$(get_time) + bucket_name="S3CMD-test%bucket%$RANDOM" + assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function setup() +{ + start_time=$(get_time) + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}" +} + +function teardown() +{ + start_time=$(get_time) + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}" +} + +function test_put_object() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_put_object_error() +{ + show "${FUNCNAME[0]}" + start_time=$(get_time) + + object_long_name=$(printf "s3cmd-test-object-%01100d" 1) + assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_put_object_multipart() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_get_object() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" + assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_get_object_error() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" + assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" + assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_get_object_multipart() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" + assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" + assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_copy_object() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + object_name1="s3cmd-test-object-$RANDOM" + object_name2="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name1}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd cp "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd info "s3://${BUCKET_NAME}/${object_name2}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name1}" "s3://${BUCKET_NAME}/${object_name2}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function test_sync_list_objects() +{ + show "${FUNCNAME[0]}" + + start_time=$(get_time) + bucket_name="s3cmd-test-bucket-$RANDOM" + object_name="s3cmd-test-object-$RANDOM" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}" + + diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1 + assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs" + assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}" + + log_success "$start_time" "${FUNCNAME[0]}" +} + +function run_test() +{ + test_make_bucket + test_make_bucket_error + + setup + + test_put_object + test_put_object_error + test_put_object_multipart + test_get_object + test_get_object_multipart + test_copy_object + test_sync_list_objects + + teardown +} + +function __init__() +{ + set -e + + S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM" + mkdir -p $S3CMD_CONFIG_DIR + S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg" + + # configure s3cmd + cat > $S3CMD_CONFIG_FILE <"$FILE_1_MB" + fi + + if [ ! -e "$FILE_65_MB" ]; then + shred -n 1 -s 65MB - >"$FILE_65_MB" + fi + + set -E + set -o pipefail + + FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")" + rv=$? + if [ $rv -ne 0 ]; then + echo "unable to get md5sum of $FILE_1_MB" + exit 1 + fi + + FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")" + rv=$? + if [ $rv -ne 0 ]; then + echo "unable to get md5sum of $FILE_65_MB" + exit 1 + fi + + set +e +} + +function main() +{ + ( run_test ) + rv=$? + + rm -fr "$S3CMD_CONFIG_FILE" + if [ -z "$MINT_MODE" ]; then + rm -fr "$WORK_DIR" "$DATA_DIR" + fi + + exit "$rv" +} + +__init__ "$@" +main "$@" diff --git a/mint/run/core/security/run.sh b/mint/run/core/security/run.sh new file mode 100755 index 000000000..9283c76ee --- /dev/null +++ b/mint/run/core/security/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2018 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/security/tls-tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/security/tls-tests.go b/mint/run/core/security/tls-tests.go new file mode 100644 index 000000000..cb059a76a --- /dev/null +++ b/mint/run/core/security/tls-tests.go @@ -0,0 +1,272 @@ +// Mint, (C) 2018 Minio, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software + +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "os" + "time" + + log "github.com/sirupsen/logrus" +) + +const testName = "TLS-tests" + +const ( + // PASS indicate that a test passed + PASS = "PASS" + // FAIL indicate that a test failed + FAIL = "FAIL" + // NA indicates that a test is not applicable + NA = "NA" +) + +func main() { + log.SetOutput(os.Stdout) + log.SetFormatter(&mintJSONFormatter{}) + log.SetLevel(log.InfoLevel) + + endpoint := os.Getenv("SERVER_ENDPOINT") + secure := os.Getenv("ENABLE_HTTPS") + if secure != "1" { + log.WithFields(log.Fields{"name:": testName, "status": NA, "message": "TLS is not enabled"}).Info() + return + } + + testTLSVersions(endpoint) + testTLSCiphers(endpoint) + testTLSEllipticCurves(endpoint) +} + +// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so. +// Tests whether the endpoint accepts TLS1.2 connections - fail if not. +func testTLSVersions(endpoint string) { + const function = "TLSVersions" + startTime := time.Now() + + // Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections + args := map[string]interface{}{ + "MinVersion": "tls.VersionSSL30", + "MaxVersion": "tls.VersionTLS11", + } + _, err := tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionSSL30, + MaxVersion: tls.VersionTLS11, + }) + if err == nil { + failureLog(function, args, startTime, "", "Endpoint accepts insecure connection", err).Error() + return + } + + // Tests whether the endpoint accepts TLS1.2 connections + args = map[string]interface{}{ + "MinVersion": "tls.VersionTLS12", + } + _, err = tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + }) + if err != nil { + failureLog(function, args, startTime, "", "Endpoint rejects secure connection", err).Error() + return + } + successLog(function, args, startTime) +} + +// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so. +// Tests whether the endpoint accepts TLS1.2 connections - fail if not. +func testTLSCiphers(endpoint string) { + const function = "TLSCiphers" + startTime := time.Now() + + // Tests whether the endpoint accepts insecure ciphers + args := map[string]interface{}{ + "MinVersion": "tls.VersionTLS12", + "CipherSuites": unsupportedCipherSuites, + } + _, err := tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: unsupportedCipherSuites, + }) + if err == nil { + failureLog(function, args, startTime, "", "Endpoint accepts insecure cipher suites", err).Error() + return + } + + // Tests whether the endpoint accepts at least one secure cipher + args = map[string]interface{}{ + "MinVersion": "tls.VersionTLS12", + "CipherSuites": supportedCipherSuites, + } + _, err = tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: supportedCipherSuites, + }) + if err != nil { + failureLog(function, args, startTime, "", "Endpoint rejects all secure cipher suites", err).Error() + return + } + + // Tests whether the endpoint accepts at least one default cipher + args = map[string]interface{}{ + "MinVersion": "tls.VersionTLS12", + "CipherSuites": nil, + } + _, err = tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: nil, // default value + }) + if err != nil { + failureLog(function, args, startTime, "", "Endpoint rejects default cipher suites", err).Error() + return + } + successLog(function, args, startTime) +} + +// Tests whether the endpoint accepts the P-384 or P-521 elliptic curve - fail if so. +// Tests whether the endpoint accepts Curve25519 or P-256 - fail if not. +func testTLSEllipticCurves(endpoint string) { + const function = "TLSEllipticCurves" + startTime := time.Now() + + // Tests whether the endpoint accepts curves using non-constant time implementations. + args := map[string]interface{}{ + "CurvePreferences": unsupportedCurves, + } + _, err := tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + CurvePreferences: unsupportedCurves, + CipherSuites: supportedCipherSuites, + }) + if err == nil { + failureLog(function, args, startTime, "", "Endpoint accepts insecure elliptic curves", err).Error() + return + } + + // Tests whether the endpoint accepts curves using constant time implementations. + args = map[string]interface{}{ + "CurvePreferences": unsupportedCurves, + } + _, err = tls.Dial("tcp", endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + CurvePreferences: supportedCurves, + CipherSuites: supportedCipherSuites, + }) + if err != nil { + failureLog(function, args, startTime, "", "Endpoint does not accept secure elliptic curves", err).Error() + return + } + successLog(function, args, startTime) +} + +func successLog(function string, args map[string]interface{}, startTime time.Time) *log.Entry { + duration := time.Since(startTime).Nanoseconds() / 1000000 + return log.WithFields(log.Fields{ + "name": testName, + "function": function, + "args": args, + "duration": duration, + "status": PASS, + }) +} + +func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { + duration := time.Since(startTime).Nanoseconds() / 1000000 + fields := log.Fields{ + "name": testName, + "function": function, + "args": args, + "duration": duration, + "status": FAIL, + "alert": alert, + "message": message, + } + if err != nil { + fields["error"] = err + } + return log.WithFields(fields) +} + +type mintJSONFormatter struct { +} + +func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { + data := make(log.Fields, len(entry.Data)) + for k, v := range entry.Data { + switch v := v.(type) { + case error: + // Otherwise errors are ignored by `encoding/json` + // https://github.com/sirupsen/logrus/issues/137 + data[k] = v.Error() + default: + data[k] = v + } + } + + serialized, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) + } + return append(serialized, '\n'), nil +} + +// Secure Go implementations of modern TLS ciphers +// The following ciphers are excluded because: +// - RC4 ciphers: RC4 is broken +// - 3DES ciphers: Because of the 64 bit blocksize of DES (Sweet32) +// - CBC-SHA256 ciphers: No countermeasures against Lucky13 timing attack +// - CBC-SHA ciphers: Legacy ciphers (SHA-1) and non-constant time +// implementation of CBC. +// (CBC-SHA ciphers can be enabled again if required) +// - RSA key exchange ciphers: Disabled because of dangerous PKCS1-v1.5 RSA +// padding scheme. See Bleichenbacher attacks. +var supportedCipherSuites = []uint16{ + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +} + +// Supported elliptic curves: Implementations are constant-time. +var supportedCurves = []tls.CurveID{tls.X25519, tls.CurveP256} + +var unsupportedCipherSuites = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, // Broken cipher + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, // Broken cipher + + // all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) + tls.TLS_RSA_WITH_RC4_128_SHA, // Broken cipher + + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. +} + +// Unsupported elliptic curves: Implementations are not constant-time. +var unsupportedCurves = []tls.CurveID{tls.CurveP384, tls.CurveP521} diff --git a/mint/run/core/worm/quick-worm-tests.go b/mint/run/core/worm/quick-worm-tests.go new file mode 100644 index 000000000..d77404d6c --- /dev/null +++ b/mint/run/core/worm/quick-worm-tests.go @@ -0,0 +1,409 @@ +/* +* Mint, (C) 2018 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" +) + +const charset = "abcdefghijklmnopqrstuvwxyz0123456789" + +var randSource *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) + +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1< " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/worm/worm 1>>"$output_log_file" 2>"$error_log_file"