This repository has been archived by the owner on Aug 1, 2018. It is now read-only.
forked from kirill-kruchkov/azure-storage-gzip-encoding
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathUtility.cs
171 lines (144 loc) · 7.17 KB
/
Utility.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Shared.Protocol;
namespace ASGE
{
static class Utility
{
public static void EnsureGzipFiles(CloudBlobContainer container, IEnumerable<string> extensions, bool inPlace, string newExtension, int cacheControlMaxAgeSeconds, bool simulate, string subpath = null)
{
Trace.TraceInformation("Enumerating files.");
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
string pathToCheck = string.IsNullOrEmpty(subpath) ? null : $"/{container.Name}/{subpath}/";
string pathToCheckFile = string.IsNullOrEmpty(subpath) ? null : $"/{container.Name}/{subpath}";
var blobInfos = container.ListBlobs(null, true, BlobListingDetails.Metadata);
Parallel.ForEach(blobInfos, (blobInfo) =>
{
CloudBlob gzipBlob = null;
CloudBlob blob = (CloudBlob)blobInfo;
// Skip other files if subpath is specified
if (pathToCheck != null && !blobInfo.Uri.LocalPath.StartsWith(pathToCheck, StringComparison.InvariantCultureIgnoreCase) && pathToCheckFile != blobInfo.Uri.LocalPath)
{
return;
}
// Only work with desired extensions
string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
if (!extensions.Contains(extension, StringComparer.OrdinalIgnoreCase))
{
return;
}
// Check if it is already done
if (inPlace)
{
if (string.Equals(blob.Properties.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase))
{
Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
return;
}
}
else
{
string gzipUrl = blob.Name + newExtension;
gzipBlob = container.GetBlockBlobReference(gzipUrl);
if (gzipBlob.Exists())
{
Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
return;
}
}
// Compress blob contents
Trace.TraceInformation("Downloading blob: " + blob.Name);
byte[] compressedBytes;
using (MemoryStream memoryStream = new MemoryStream())
{
using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
using (var blobStream = blob.OpenRead())
{
blobStream.CopyTo(gzipStream);
}
compressedBytes = memoryStream.ToArray();
}
// Blob to write to
CloudBlockBlob destinationBlob;
if (inPlace)
{
destinationBlob = (CloudBlockBlob)blob;
}
else
{
destinationBlob = (CloudBlockBlob)gzipBlob;
}
if (simulate)
{
Trace.TraceInformation("NOT writing blob, due to simulation: " + blob.Name);
}
else
{
// Upload the compressed bytes to the new blob
Trace.TraceInformation("Writing blob: " + blob.Name);
destinationBlob.UploadFromByteArray(compressedBytes, 0, compressedBytes.Length);
// Set the blob headers
Trace.TraceInformation("Configuring headers");
destinationBlob.Properties.CacheControl = cacheControlHeader;
destinationBlob.Properties.ContentType = blob.Properties.ContentType;
destinationBlob.Properties.ContentEncoding = "gzip";
destinationBlob.SetProperties();
}
});
}
public static void SetWildcardCorsOnBlobService(this CloudStorageAccount storageAccount)
{
storageAccount.SetCORSPropertiesOnBlobService(cors =>
{
var wildcardRule = new CorsRule() { AllowedMethods = CorsHttpMethods.Get, AllowedOrigins = { "*" } };
cors.CorsRules.Clear();
cors.CorsRules.Add(wildcardRule);
return cors;
});
}
public static void SetCORSPropertiesOnBlobService(this CloudStorageAccount storageAccount,
Func<CorsProperties, CorsProperties> alterCorsRules)
{
Trace.TraceInformation("Configuring CORS.");
if (storageAccount == null || alterCorsRules == null) throw new ArgumentNullException();
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ServiceProperties serviceProperties = blobClient.GetServiceProperties();
serviceProperties.Cors = alterCorsRules(serviceProperties.Cors) ?? new CorsProperties();
blobClient.SetServiceProperties(serviceProperties);
}
public static void ApplyCacheControlHeader(CloudBlobContainer container, IEnumerable<string> extensions, bool inPlace, string newExtension, int cacheControlMaxAgeSeconds, bool simulate, string subpath = null)
{
Trace.TraceInformation("Enumerating files.");
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
string pathToCheck = string.IsNullOrEmpty(subpath) ? null : $"/{container.Name}/{subpath}/";
string pathToCheckFile = string.IsNullOrEmpty(subpath) ? null : $"/{container.Name}/{subpath}";
var blobInfos = container.ListBlobs(null, true, BlobListingDetails.Metadata);
Parallel.ForEach(blobInfos, (blobInfo) =>
{
// Skip other files if subpath is specified
if (pathToCheck != null && !blobInfo.Uri.LocalPath.StartsWith(pathToCheck, StringComparison.InvariantCultureIgnoreCase) && pathToCheckFile != blobInfo.Uri.LocalPath)
{
return;
}
// Only work with desired extensions
string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
if (!extensions.Contains(extension, StringComparer.OrdinalIgnoreCase))
{
return;
}
CloudBlob blob = (CloudBlob)blobInfo;
Trace.TraceInformation("Configuring headers");
blob.Properties.CacheControl = cacheControlHeader;
blob.SetProperties();
});
}
}
}