Update azure dependency. Add contianer registry.

This commit is contained in:
Brendan Burns
2016-12-06 22:14:21 -08:00
parent bb792b17e4
commit 8d90cadc0b
79 changed files with 5225 additions and 2031 deletions

View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
@@ -256,6 +257,23 @@ const (
blobCopyStatusFailed = "failed"
)
// lease constants.
const (
leaseHeaderPrefix = "x-ms-lease-"
leaseID = "x-ms-lease-id"
leaseAction = "x-ms-lease-action"
leaseBreakPeriod = "x-ms-lease-break-period"
leaseDuration = "x-ms-lease-duration"
leaseProposedID = "x-ms-proposed-lease-id"
leaseTime = "x-ms-lease-time"
acquireLease = "acquire"
renewLease = "renew"
changeLease = "change"
releaseLease = "release"
breakLease = "break"
)
// BlockListType is used to filter out types of blocks in a Get Blocks List call
// for a block blob.
//
@@ -284,6 +302,65 @@ const (
ContainerAccessTypeContainer ContainerAccessType = "container"
)
// ContainerAccessOptions are used when setting ACLs of containers (after creation)
type ContainerAccessOptions struct {
ContainerAccess ContainerAccessType
Timeout int
LeaseID string
}
// AccessPolicyDetails are used for SETTING policies
type AccessPolicyDetails struct {
ID string
StartTime time.Time
ExpiryTime time.Time
CanRead bool
CanWrite bool
CanDelete bool
}
// ContainerPermissions is used when setting permissions and Access Policies for containers.
type ContainerPermissions struct {
AccessOptions ContainerAccessOptions
AccessPolicy AccessPolicyDetails
}
// AccessPolicyDetailsXML has specifics about an access policy
// annotated with XML details.
type AccessPolicyDetailsXML struct {
StartTime time.Time `xml:"Start"`
ExpiryTime time.Time `xml:"Expiry"`
Permission string `xml:"Permission"`
}
// SignedIdentifier is a wrapper for a specific policy
type SignedIdentifier struct {
ID string `xml:"Id"`
AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"`
}
// SignedIdentifiers part of the response from GetPermissions call.
type SignedIdentifiers struct {
SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"`
}
// AccessPolicy is the response type from the GetPermissions call.
type AccessPolicy struct {
SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"`
}
// ContainerAccessResponse is returned for the GetContainerPermissions function.
// This contains both the permission and access policy for the container.
type ContainerAccessResponse struct {
ContainerAccess ContainerAccessType
AccessPolicy SignedIdentifiers
}
// ContainerAccessHeader references header used when setting/getting container ACL
const (
ContainerAccessHeader string = "x-ms-blob-public-access"
)
// Maximum sizes (per REST API) for various concepts
const (
MaxBlobBlockSize = 4 * 1024 * 1024
@@ -399,7 +476,7 @@ func (b BlobStorageClient) createContainer(name string, access ContainerAccessTy
headers := b.client.getStandardHeaders()
if access != "" {
headers["x-ms-blob-public-access"] = string(access)
headers[ContainerAccessHeader] = string(access)
}
return b.client.exec(verb, uri, headers, nil)
}
@@ -421,6 +498,101 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) {
return false, err
}
// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391.aspx
func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) (err error) {
params := url.Values{
"restype": {"container"},
"comp": {"acl"},
}
if containerPermissions.AccessOptions.Timeout > 0 {
params.Add("timeout", strconv.Itoa(containerPermissions.AccessOptions.Timeout))
}
uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params)
headers := b.client.getStandardHeaders()
if containerPermissions.AccessOptions.ContainerAccess != "" {
headers[ContainerAccessHeader] = string(containerPermissions.AccessOptions.ContainerAccess)
}
if containerPermissions.AccessOptions.LeaseID != "" {
headers[leaseID] = containerPermissions.AccessOptions.LeaseID
}
// generate the XML for the SharedAccessSignature if required.
accessPolicyXML, err := generateAccessPolicy(containerPermissions.AccessPolicy)
if err != nil {
return err
}
var resp *storageResponse
if accessPolicyXML != "" {
headers["Content-Length"] = strconv.Itoa(len(accessPolicyXML))
resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML))
} else {
resp, err = b.client.exec("PUT", uri, headers, nil)
}
if err != nil {
return err
}
if resp != nil {
defer func() {
err = resp.body.Close()
}()
if resp.statusCode != http.StatusOK {
return errors.New("Unable to set permissions")
}
}
return nil
}
// GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx
// If timeout is 0 then it will not be passed to Azure
// leaseID will only be passed to Azure if populated
// Returns permissionResponse which is combined permissions and AccessPolicy
func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (permissionResponse *ContainerAccessResponse, err error) {
params := url.Values{"restype": {"container"},
"comp": {"acl"}}
if timeout > 0 {
params.Add("timeout", strconv.Itoa(timeout))
}
uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params)
headers := b.client.getStandardHeaders()
if leaseID != "" {
headers[leaseID] = leaseID
}
resp, err := b.client.exec("GET", uri, headers, nil)
if err != nil {
return nil, err
}
// containerAccess. Blob, Container, empty
containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader))
defer func() {
err = resp.body.Close()
}()
var out AccessPolicy
err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList)
if err != nil {
return nil, err
}
permissionResponse = &ContainerAccessResponse{}
permissionResponse.AccessPolicy = out.SignedIdentifiersList
permissionResponse.ContainerAccess = ContainerAccessType(containerAccess)
return permissionResponse, nil
}
// DeleteContainer deletes the container with given name on the storage
// account. If the container does not exist returns error.
//
@@ -560,6 +732,174 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string, extr
return resp, err
}
// leasePut is common PUT code for the various aquire/release/break etc functions.
func (b BlobStorageClient) leaseCommonPut(container string, name string, headers map[string]string, expectedStatus int) (http.Header, error) {
params := url.Values{"comp": {"lease"}}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err := checkRespCode(resp.statusCode, []int{expectedStatus}); err != nil {
return nil, err
}
return resp.headers, nil
}
// SnapshotBlob creates a snapshot for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691971.aspx
func (b BlobStorageClient) SnapshotBlob(container string, name string, timeout int, extraHeaders map[string]string) (snapshotTimestamp *time.Time, err error) {
headers := b.client.getStandardHeaders()
params := url.Values{"comp": {"snapshot"}}
if timeout > 0 {
params.Add("timeout", strconv.Itoa(timeout))
}
for k, v := range extraHeaders {
headers[k] = v
}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
return nil, err
}
if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil {
return nil, err
}
snapshotResponse := resp.headers.Get(http.CanonicalHeaderKey("x-ms-snapshot"))
if snapshotResponse != "" {
snapshotTimestamp, err := time.Parse(time.RFC3339, snapshotResponse)
if err != nil {
return nil, err
}
return &snapshotTimestamp, nil
}
return nil, errors.New("Snapshot not created")
}
// AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
// returns leaseID acquired
func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (returnedLeaseID string, err error) {
headers := b.client.getStandardHeaders()
headers[leaseAction] = acquireLease
if leaseTimeInSeconds > 0 {
headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds)
}
if proposedLeaseID != "" {
headers[leaseProposedID] = proposedLeaseID
}
respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusCreated)
if err != nil {
return "", err
}
returnedLeaseID = respHeaders.Get(http.CanonicalHeaderKey(leaseID))
if returnedLeaseID != "" {
return returnedLeaseID, nil
}
return "", errors.New("LeaseID not returned")
}
// BreakLease breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
// Returns the timeout remaining in the lease in seconds
func (b BlobStorageClient) BreakLease(container string, name string) (breakTimeout int, err error) {
headers := b.client.getStandardHeaders()
headers[leaseAction] = breakLease
return b.breakLeaseCommon(container, name, headers)
}
// BreakLeaseWithBreakPeriod breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
// breakPeriodInSeconds is used to determine how long until new lease can be created.
// Returns the timeout remaining in the lease in seconds
func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name string, breakPeriodInSeconds int) (breakTimeout int, err error) {
headers := b.client.getStandardHeaders()
headers[leaseAction] = breakLease
headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds)
return b.breakLeaseCommon(container, name, headers)
}
// breakLeaseCommon is common code for both version of BreakLease (with and without break period)
func (b BlobStorageClient) breakLeaseCommon(container string, name string, headers map[string]string) (breakTimeout int, err error) {
respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusAccepted)
if err != nil {
return 0, err
}
breakTimeoutStr := respHeaders.Get(http.CanonicalHeaderKey(leaseTime))
if breakTimeoutStr != "" {
breakTimeout, err = strconv.Atoi(breakTimeoutStr)
if err != nil {
return 0, err
}
}
return breakTimeout, nil
}
// ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
// Returns the new LeaseID acquired
func (b BlobStorageClient) ChangeLease(container string, name string, currentLeaseID string, proposedLeaseID string) (newLeaseID string, err error) {
headers := b.client.getStandardHeaders()
headers[leaseAction] = changeLease
headers[leaseID] = currentLeaseID
headers[leaseProposedID] = proposedLeaseID
respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
if err != nil {
return "", err
}
newLeaseID = respHeaders.Get(http.CanonicalHeaderKey(leaseID))
if newLeaseID != "" {
return newLeaseID, nil
}
return "", errors.New("LeaseID not returned")
}
// ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
func (b BlobStorageClient) ReleaseLease(container string, name string, currentLeaseID string) error {
headers := b.client.getStandardHeaders()
headers[leaseAction] = releaseLease
headers[leaseID] = currentLeaseID
_, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
if err != nil {
return err
}
return nil
}
// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
func (b BlobStorageClient) RenewLease(container string, name string, currentLeaseID string) error {
headers := b.client.getStandardHeaders()
headers[leaseAction] = renewLease
headers[leaseID] = currentLeaseID
_, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
if err != nil {
return err
}
return nil
}
// GetBlobProperties provides various information about the specified
// blob. See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx
func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) {
@@ -961,15 +1301,20 @@ func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte, ext
//
// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx
func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error {
copyID, err := b.startBlobCopy(container, name, sourceBlob)
copyID, err := b.StartBlobCopy(container, name, sourceBlob)
if err != nil {
return err
}
return b.waitForBlobCopy(container, name, copyID)
return b.WaitForBlobCopy(container, name, copyID)
}
func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (string, error) {
// StartBlobCopy starts a blob copy operation.
// sourceBlob parameter must be a canonical URL to the blob (can be
// obtained using GetBlobURL method.)
//
// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx
func (b BlobStorageClient) StartBlobCopy(container, name, sourceBlob string) (string, error) {
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})
headers := b.client.getStandardHeaders()
@@ -992,7 +1337,39 @@ func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (st
return copyID, nil
}
func (b BlobStorageClient) waitForBlobCopy(container, name, copyID string) error {
// AbortBlobCopy aborts a BlobCopy which has already been triggered by the StartBlobCopy function.
// copyID is generated from StartBlobCopy function.
// currentLeaseID is required IF the destination blob has an active lease on it.
// As defined in https://msdn.microsoft.com/en-us/library/azure/jj159098.aspx
func (b BlobStorageClient) AbortBlobCopy(container, name, copyID, currentLeaseID string, timeout int) error {
params := url.Values{"comp": {"copy"}, "copyid": {copyID}}
if timeout > 0 {
params.Add("timeout", strconv.Itoa(timeout))
}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
headers := b.client.getStandardHeaders()
headers["x-ms-copy-action"] = "abort"
if currentLeaseID != "" {
headers[leaseID] = currentLeaseID
}
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
return err
}
defer resp.body.Close()
if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return nil
}
// WaitForBlobCopy loops until a BlobCopy operation is completed (or fails with error)
func (b BlobStorageClient) WaitForBlobCopy(container, name, copyID string) error {
for {
props, err := b.GetBlobProperties(container, name)
if err != nil {
@@ -1036,10 +1413,12 @@ func (b BlobStorageClient) DeleteBlob(container, name string, extraHeaders map[s
// See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx
func (b BlobStorageClient) DeleteBlobIfExists(container, name string, extraHeaders map[string]string) (bool, error) {
resp, err := b.deleteBlob(container, name, extraHeaders)
if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) {
return resp.statusCode == http.StatusAccepted, nil
if resp != nil {
defer resp.body.Close()
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
defer resp.body.Close()
return false, err
}
@@ -1065,17 +1444,18 @@ func pathForBlob(container, name string) string {
return fmt.Sprintf("/%s/%s", container, name)
}
// GetBlobSASURI creates an URL to the specified blob which contains the Shared
// Access Signature with specified permissions and expiration time.
// GetBlobSASURIWithSignedIPAndProtocol creates an URL to the specified blob which contains the Shared
// Access Signature with specified permissions and expiration time. Also includes signedIPRange and allowed procotols.
// If old API version is used but no signedIP is passed (ie empty string) then this should still work.
// We only populate the signedIP when it non-empty.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx
func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) {
func (b BlobStorageClient) GetBlobSASURIWithSignedIPAndProtocol(container, name string, expiry time.Time, permissions string, signedIPRange string, HTTPSOnly bool) (string, error) {
var (
signedPermissions = permissions
blobURL = b.GetBlobURL(container, name)
)
canonicalizedResource, err := b.client.buildCanonicalizedResource(blobURL)
if err != nil {
return "", err
}
@@ -1087,7 +1467,6 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
// We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component).
canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1)
canonicalizedResource, err = url.QueryUnescape(canonicalizedResource)
if err != nil {
return "", err
@@ -1096,7 +1475,11 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
signedExpiry := expiry.UTC().Format(time.RFC3339)
signedResource := "b"
stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions)
protocols := "https,http"
if HTTPSOnly {
protocols = "https"
}
stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions, signedIPRange, protocols)
if err != nil {
return "", err
}
@@ -1110,6 +1493,13 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
"sig": {sig},
}
if b.client.apiVersion >= "2015-04-05" {
sasParams.Add("spr", protocols)
if signedIPRange != "" {
sasParams.Add("sip", signedIPRange)
}
}
sasURL, err := url.Parse(blobURL)
if err != nil {
return "", err
@@ -1118,16 +1508,89 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
return sasURL.String(), nil
}
func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string) (string, error) {
// GetBlobSASURI creates an URL to the specified blob which contains the Shared
// Access Signature with specified permissions and expiration time.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx
func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) {
url, err := b.GetBlobSASURIWithSignedIPAndProtocol(container, name, expiry, permissions, "", false)
return url, err
}
func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string, signedIP string, protocols string) (string, error) {
var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string
if signedVersion >= "2015-02-21" {
canonicalizedResource = "/blob" + canonicalizedResource
}
// https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12
if signedVersion >= "2015-04-05" {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
}
// reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
if signedVersion >= "2013-08-15" {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
}
return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15")
}
func generatePermissions(accessPolicy AccessPolicyDetails) (permissions string) {
// generate the permissions string (rwd).
// still want the end user API to have bool flags.
permissions = ""
if accessPolicy.CanRead {
permissions += "r"
}
if accessPolicy.CanWrite {
permissions += "w"
}
if accessPolicy.CanDelete {
permissions += "d"
}
return permissions
}
// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the
// AccessPolicy struct which will get converted to XML.
func convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) SignedIdentifiers {
return SignedIdentifiers{
SignedIdentifiers: []SignedIdentifier{
{
ID: accessPolicy.ID,
AccessPolicy: AccessPolicyDetailsXML{
StartTime: accessPolicy.StartTime.UTC().Round(time.Second),
ExpiryTime: accessPolicy.ExpiryTime.UTC().Round(time.Second),
Permission: generatePermissions(accessPolicy),
},
},
},
}
}
// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions.
func generateAccessPolicy(accessPolicy AccessPolicyDetails) (accessPolicyXML string, err error) {
if accessPolicy.ID != "" {
signedIdentifiers := convertAccessPolicyToXMLStructs(accessPolicy)
body, _, err := xmlMarshal(signedIdentifiers)
if err != nil {
return "", err
}
xmlByteArray, err := ioutil.ReadAll(body)
if err != nil {
return "", err
}
accessPolicyXML = string(xmlByteArray)
return accessPolicyXML, nil
}
return "", nil
}