Create a Timer Triggered .NET Azure function to scans all blobs in the target container and report the counts of blobs grouped by different index tags. For example, the total number of blobs that have the following index tags defined: “BackupType” = ‘FULL’ and “KeepForLTRMonthly” = ‘NO’ and “KeepForLTRYearly” = ‘NO’
Enable “RunONStartup” to run the function whenever the function App restarts. Since we don’t want to run this function repeatedly, it should be disable once the task is done, but if it’s needed, the function can be invoked by a timer defined in the function code.
1. In Solution Explorer, right click on AzureBlobIndexTags, select Add, then select New Azure Function. On Add New Item – AzureBlobIndexTags window, select Azure Function, then click Add button.
2. In Visual Studio, rename the automatically created function Function1.cs to VerifyIndexTagsOnExistingBlobs.cs and replace all its content with the following code block
using System;
using Microsoft.Azure.WebJobs;
using Microsoft.Extensions.Logging;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace AzureBlobIndexTags
{
public static class VerifyIndexTagsOnExistingBlobs
{
static string connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
static string containerName = Environment.GetEnvironmentVariable("ContainerName");
[FunctionName("VerifyIndexTagsOnExitingBlobs")]
// [Timeout("-1")]
public async static Task RunAsync([TimerTrigger("0 5 22 30 9 6", RunOnStartup = true)] TimerInfo myTimer, ILogger log)
{
log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
string blobIndexTagKey01 = Environment.GetEnvironmentVariable("BITagKey01");
string blobIndexTagKey01Value = Environment.GetEnvironmentVariable("BITagKey01DefaultValue");
string blobIndexTagKey02 = Environment.GetEnvironmentVariable("BITagKey02");
string blobIndexTagKey02Value = Environment.GetEnvironmentVariable("BITagKey02DefaultValue");
string blobIndexTagKey03 = Environment.GetEnvironmentVariable("BITagKey03");
string blobIndexTagKey03Value = Environment.GetEnvironmentVariable("BITagKey03DefaultValue");
BlobServiceClient serviceClient = new BlobServiceClient(connectionString);
BlobContainerClient container = serviceClient.GetBlobContainerClient(containerName);
// Blob index queries and selection
String QueryLogFiles = $@"@container = '{containerName}' AND ""BackupType"" = 'LOG'";
String QueryDiffBkpFiles = $@"@container = '{containerName}' AND ""BackupType"" = 'DIFF'";
String QueryFullBkpFilesNoLTR = $@"@container = '{containerName}' AND ""BackupType"" = 'FULL' AND ""KeepForLTRMonthly"" = 'NO' AND ""KeepForLTRYearly"" = 'NO'";
String QueryFullBkpFilesLTRMonthly = $@"@container = '{containerName}' AND ""BackupType"" = 'FULL' AND ""KeepForLTRMonthly"" = 'YES' AND ""KeepForLTRYearly"" = 'NO'";
String QueryFullBkpFilesMonthlyYearly = $@"@container = '{containerName}' AND ""BackupType"" = 'FULL' AND ""KeepForLTRMonthly"" = 'YES' AND ""KeepForLTRYearly"" = 'YES'";
String QueryFullBkpFilesLTRYearly = $@"@container = '{containerName}' AND ""BackupType"" = 'FULL' AND ""KeepForLTRMonthly"" = 'NO' AND ""KeepForLTRYearly"" = 'YES'";
String QueryOtherFiles = $@"@container = '{containerName}' AND ""BackupType"" = 'Others'";
var watch = new System.Diagnostics.Stopwatch();
watch.Start();
List<TaggedBlobItem> logBkpFiles = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryLogFiles))
{
logBkpFiles.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryLogFiles}");
log.LogInformation($"The Number of blos with index tag 'LOG' is {logBkpFiles.Count}");
List<TaggedBlobItem> diffBkpFiles = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryDiffBkpFiles))
{
diffBkpFiles.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryDiffBkpFiles}");
log.LogInformation($"The Number of blos with index tag 'DIFF' is {diffBkpFiles.Count}");
List<TaggedBlobItem> fullBkpFilesNoLTR = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryFullBkpFilesNoLTR))
{
fullBkpFilesNoLTR.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryFullBkpFilesNoLTR}");
log.LogInformation($"The Number of blos with index tag 'FULL' and not required for LTR is {fullBkpFilesNoLTR.Count}");
List<TaggedBlobItem> fullBkpFilesLTRMonthly = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryFullBkpFilesLTRMonthly))
{
fullBkpFilesLTRMonthly.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryFullBkpFilesLTRMonthly}");
log.LogInformation($"The Number of blos with index tag 'FULL' and required for monthly LTR is {fullBkpFilesLTRMonthly.Count}");
List<TaggedBlobItem> fullBkpFilesLTRYearly = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryFullBkpFilesLTRYearly))
{
fullBkpFilesLTRYearly.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryFullBkpFilesLTRYearly}");
log.LogInformation($"The Number of blos with index tag 'FULL' and required for yearly LTR is {fullBkpFilesLTRYearly.Count}");
List<TaggedBlobItem> fullBkpFilesMonthlyYearly = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryFullBkpFilesMonthlyYearly))
{
fullBkpFilesMonthlyYearly.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryFullBkpFilesMonthlyYearly}");
log.LogInformation($"The Number of blos with index tag 'FULL' and required for both monthly and yearly LTR is {fullBkpFilesMonthlyYearly.Count}");
List<TaggedBlobItem> otherFiles = new List<TaggedBlobItem>();
await foreach (TaggedBlobItem taggedBlobItem in serviceClient.FindBlobsByTagsAsync(QueryOtherFiles))
{
otherFiles.Add(taggedBlobItem);
}
log.LogInformation($"The query is: {QueryOtherFiles}");
log.LogInformation($"The Number of blos with index tag 'Others' is {otherFiles.Count}");
int numberOfBlobsWithoutTag = 0;
await foreach (BlobItem blobItem in container.GetBlobsAsync(BlobTraits.All))
{
if (blobItem.Tags is null)
{
// log.LogWarning($"The blob '{blobItem.Name}' doesn't have tag");
numberOfBlobsWithoutTag += 1;
}
}
log.LogInformation($"The total number of blobs that don't have index tags is {numberOfBlobsWithoutTag}");
watch.Stop();
Console.WriteLine($"Execution Time: {watch.ElapsedMilliseconds} ms");
log.LogInformation($"Execution Time: {watch.ElapsedMilliseconds} ms");
}
}
}3. Edit local.settings.json file to add a new line and modify one of the existing lines to disable the function AutoAddIndexTags and AddIndexTagsOnExistingBlobs so that we can test to run this new function VerifyIndexTagsOnExistingBlobs alone in the local runtime environment.
“AzureWebJobs.VerifyIndexTagsOnExistingBlobs.Disabled”: false, “AzureWebJobs.AddIndexTagsOnExistingBlobs.Disabled”: true, “AzureWebJobs.AutoAddIndexTags.Disabled”: true,
4. Manually run AzureBlobIndexTags in Visual Studio and observe the func.exe (Azure Functions Core Tools) window to verify the function run successfully and show reports about the index tags on all the existing blobs in the container.
5. Stop the function app AzureBlobIndexTags
6. In Solution Explorer, right click on AzureBlobIndexTags, select Publish, then click Publish.
7. Login Azure portal, find the function VerifyIndexTagsOnExitingBlobs, go the function Monitor window, click on the latest invocation trace to show the Invocation Details window. As you can see, the function has been triggered and ran successfully.
8. We now have successfully created and deployed the C# Azure function VerifyIndexTagsOnExitingBlobs in the function App AzureBlobIndexTags20211029 in Azure.
Comments