s3 binary cache: support specifying an endpoint
Works for uploading and not downloading.
This commit is contained in:
parent
3193f5ff3e
commit
49a53c1d3f
5 changed files with 33 additions and 8 deletions
|
@ -118,12 +118,29 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
|
||||||
</para>
|
</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
<varlistentry><term><literal>endpoint</literal></term>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
The URL to your S3-compatible service, for when not using
|
||||||
|
Amazon S3. Do not specify this value if you're using Amazon
|
||||||
|
S3.
|
||||||
|
</para>
|
||||||
|
<note><para>This endpoint must support HTTPS and will use
|
||||||
|
path-based addressing instead of virtual host based
|
||||||
|
addressing.</para></note>
|
||||||
|
</listitem>
|
||||||
|
</varlistentry>
|
||||||
</variablelist>
|
</variablelist>
|
||||||
|
|
||||||
<example><title>Uploading with non-default credential profile for Amazon S3</title>
|
<example><title>Uploading with non-default credential profile for Amazon S3</title>
|
||||||
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
|
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
|
||||||
</example>
|
</example>
|
||||||
|
|
||||||
|
<example><title>Uploading to an S3-Compatible Binary Cache</title>
|
||||||
|
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&endpoint=minio.example.com</command></para>
|
||||||
|
</example>
|
||||||
|
|
||||||
<para>The user writing to the bucket will need to perform the
|
<para>The user writing to the bucket will need to perform the
|
||||||
following actions against the bucket:</para>
|
following actions against the bucket:</para>
|
||||||
|
|
||||||
|
|
|
@ -598,7 +598,7 @@ struct CurlDownloader : public Downloader
|
||||||
// FIXME: do this on a worker thread
|
// FIXME: do this on a worker thread
|
||||||
try {
|
try {
|
||||||
#ifdef ENABLE_S3
|
#ifdef ENABLE_S3
|
||||||
S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
|
S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
|
||||||
auto slash = request.uri.find('/', 5);
|
auto slash = request.uri.find('/', 5);
|
||||||
if (slash == std::string::npos)
|
if (slash == std::string::npos)
|
||||||
throw nix::Error("bad S3 URI '%s'", request.uri);
|
throw nix::Error("bad S3 URI '%s'", request.uri);
|
||||||
|
|
|
@ -84,8 +84,8 @@ static void initAWS()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
S3Helper::S3Helper(const std::string & profile, const std::string & region)
|
S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
|
||||||
: config(makeConfig(region))
|
: config(makeConfig(region, endpoint))
|
||||||
, client(make_ref<Aws::S3::S3Client>(
|
, client(make_ref<Aws::S3::S3Client>(
|
||||||
profile == ""
|
profile == ""
|
||||||
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
||||||
|
@ -99,7 +99,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
|
||||||
#else
|
#else
|
||||||
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
||||||
#endif
|
#endif
|
||||||
false))
|
endpoint.empty()))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,11 +116,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
|
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
|
||||||
{
|
{
|
||||||
initAWS();
|
initAWS();
|
||||||
auto res = make_ref<Aws::Client::ClientConfiguration>();
|
auto res = make_ref<Aws::Client::ClientConfiguration>();
|
||||||
res->region = region;
|
res->region = region;
|
||||||
|
if (!endpoint.empty()) {
|
||||||
|
res->endpointOverride = endpoint;
|
||||||
|
}
|
||||||
res->requestTimeoutMs = 600 * 1000;
|
res->requestTimeoutMs = 600 * 1000;
|
||||||
res->retryStrategy = std::make_shared<RetryStrategy>();
|
res->retryStrategy = std::make_shared<RetryStrategy>();
|
||||||
res->caFile = settings.caFile;
|
res->caFile = settings.caFile;
|
||||||
|
@ -170,6 +173,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
{
|
{
|
||||||
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
|
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||||
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||||
|
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
||||||
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
|
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||||
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
|
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
|
||||||
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
|
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
|
||||||
|
@ -186,7 +190,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
const Params & params, const std::string & bucketName)
|
const Params & params, const std::string & bucketName)
|
||||||
: S3BinaryCacheStore(params)
|
: S3BinaryCacheStore(params)
|
||||||
, bucketName(bucketName)
|
, bucketName(bucketName)
|
||||||
, s3Helper(profile, region)
|
, s3Helper(profile, region, endpoint)
|
||||||
{
|
{
|
||||||
diskCache = getNarInfoDiskCache();
|
diskCache = getNarInfoDiskCache();
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,9 +14,9 @@ struct S3Helper
|
||||||
ref<Aws::Client::ClientConfiguration> config;
|
ref<Aws::Client::ClientConfiguration> config;
|
||||||
ref<Aws::S3::S3Client> client;
|
ref<Aws::S3::S3Client> client;
|
||||||
|
|
||||||
S3Helper(const std::string & profile, const std::string & region);
|
S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
|
||||||
|
|
||||||
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
|
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
|
||||||
|
|
||||||
struct DownloadResult
|
struct DownloadResult
|
||||||
{
|
{
|
||||||
|
|
|
@ -72,6 +72,10 @@ struct CmdCopy : StorePathsCommand
|
||||||
"To populate the current folder build output to a S3 binary cache:",
|
"To populate the current folder build output to a S3 binary cache:",
|
||||||
"nix copy --to s3://my-bucket?region=eu-west-1"
|
"nix copy --to s3://my-bucket?region=eu-west-1"
|
||||||
},
|
},
|
||||||
|
Example{
|
||||||
|
"To populate the current folder build output to an S3-compatible binary cache:",
|
||||||
|
"nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
|
||||||
|
},
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue