diff --git a/CleanupOptionsBinder.cs b/CleanupOptionsBinder.cs new file mode 100644 index 0000000..d334703 --- /dev/null +++ b/CleanupOptionsBinder.cs @@ -0,0 +1,64 @@ +using AMSMigrate.Contracts; +using System.CommandLine; +using System.CommandLine.Binding; + +namespace AMSMigrate +{ + internal class CleanupOptionsBinder : BinderBase + { + private readonly Option _sourceAccount = new Option( + aliases: new[] { "--source-account-name", "-n" }, + description: "Azure Media Services Account.") + { + IsRequired = true, + Arity = ArgumentArity.ExactlyOne + }; + + private readonly Option _filter = new Option( + aliases: new[] { "--resource-filter", "-f" }, + description: @"An ODATA condition to filter the resources only when the source account is for media service. +e.g.: ""name eq 'asset1'"" to match an asset with name 'asset1'. +Visit https://learn.microsoft.com/en-us/azure/media-services/latest/filter-order-page-entities-how-to for more information.") + { + Arity = ArgumentArity.ZeroOrOne + }; + + private readonly Option _isForceCleanUpAsset = new( + aliases: new[] {"--force-cleanup", "-x"}, + () => false, + description: @"Force the cleanup of the selected input assets no matter what migration status is.") + { + IsRequired = false + }; + + private readonly Option _isCleanUpAccount = new( + aliases: new[] {"--cleanup-account", "-ax"}, + () => false, + description: @"Delete the whole ams account.") + { + IsRequired = false + }; + + public CleanupOptions GetValue(BindingContext context) => GetBoundValue(context); + + public Command GetCommand(string name, string description) + { + var command = new Command(name, description); + command.AddOption(_sourceAccount); + command.AddOption(_filter); + command.AddOption(_isForceCleanUpAsset); + command.AddOption(_isCleanUpAccount); + return command; + } + + protected override CleanupOptions GetBoundValue(BindingContext bindingContext) + { + return new CleanupOptions( + bindingContext.ParseResult.GetValueForOption(_sourceAccount)!, + bindingContext.ParseResult.GetValueForOption(_filter), + bindingContext.ParseResult.GetValueForOption(_isForceCleanUpAsset), + bindingContext.ParseResult.GetValueForOption(_isCleanUpAccount) + ); + } + } +} diff --git a/Program.cs b/Program.cs index 78b3ed7..6ef0224 100644 --- a/Program.cs +++ b/Program.cs @@ -1,4 +1,5 @@ -using AMSMigrate.Ams; +using AMSMigrate.ams; +using AMSMigrate.Ams; using AMSMigrate.Azure; using AMSMigrate.Contracts; using AMSMigrate.Local; @@ -35,7 +36,8 @@ public static async Task Main(string[] args) amsmigrate analyze -s -g -n This will analyze the given media account and produce a summary report."); rootCommand.Add(analyzeCommand); - analyzeCommand.SetHandler(async context => { + analyzeCommand.SetHandler(async context => + { var analysisOptions = analysisOptionsBinder.GetValue(context.BindingContext); await AnalyzeAssetsAsync(context, analysisOptions, context.GetCancellationToken()); }); @@ -54,36 +56,53 @@ amsmigrate assets -s -g -n await MigrateAssetsAsync(context, assetOptions, context.GetCancellationToken()); }); -// disable storage migrate option until ready -/* - var storageOptionsBinder = new StorageOptionsBinder(); - var storageCommand = storageOptionsBinder.GetCommand("storage", @"Directly migrate the assets from the storage account. -Doesn't require the Azure media services to be running. -Examples: -amsmigrate storage -s -g -n -o -t path-template -"); - rootCommand.Add(storageCommand); - storageCommand.SetHandler(async context => - { - var globalOptions = globalOptionsBinder.GetValue(context.BindingContext); - var storageOptions = storageOptionsBinder.GetValue(context.BindingContext); - await MigrateStorageAsync(globalOptions, storageOptions, context.GetCancellationToken()); - }); -*/ - -// disable key migrate option until ready -/* - var keyOptionsBinder = new KeyOptionsBinder(); - var keysCommand = keyOptionsBinder.GetCommand(); - rootCommand.Add(keysCommand); - keysCommand.SetHandler( + + var cleanupOptionsBinder = new CleanupOptionsBinder(); + var cleanupCommand = cleanupOptionsBinder.GetCommand("cleanup", @"Do the cleanup of AMS account or Storage account +Examples to cleanup account: +cleanup -s -g -n -ax true +This command forcefully removes the Azure Media Services (AMS) account. +Examples to cleanup asset: +cleanup -s -g -n -x true +This command forcefully removes all assets in the given account."); + rootCommand.Add(cleanupCommand); + cleanupCommand.SetHandler( async context => { - var globalOptions = globalOptionsBinder.GetValue(context.BindingContext); - var keyOptions = keyOptionsBinder.GetValue(context.BindingContext); - await MigrateKeysAsync(globalOptions, keyOptions, context.GetCancellationToken()); + var cleanupOptions = cleanupOptionsBinder.GetValue(context.BindingContext); + await CleanupAsync(context, cleanupOptions, context.GetCancellationToken()); }); -*/ + + // disable storage migrate option until ready + /* + var storageOptionsBinder = new StorageOptionsBinder(); + var storageCommand = storageOptionsBinder.GetCommand("storage", @"Directly migrate the assets from the storage account. + Doesn't require the Azure media services to be running. + Examples: + amsmigrate storage -s -g -n -o -t path-template + "); + rootCommand.Add(storageCommand); + storageCommand.SetHandler(async context => + { + var globalOptions = globalOptionsBinder.GetValue(context.BindingContext); + var storageOptions = storageOptionsBinder.GetValue(context.BindingContext); + await MigrateStorageAsync(globalOptions, storageOptions, context.GetCancellationToken()); + }); + */ + + // disable key migrate option until ready + /* + var keyOptionsBinder = new KeyOptionsBinder(); + var keysCommand = keyOptionsBinder.GetCommand(); + rootCommand.Add(keysCommand); + keysCommand.SetHandler( + async context => + { + var globalOptions = globalOptionsBinder.GetValue(context.BindingContext); + var keyOptions = keyOptionsBinder.GetValue(context.BindingContext); + await MigrateKeysAsync(globalOptions, keyOptions, context.GetCancellationToken()); + }); + */ var parser = new CommandLineBuilder(rootCommand) .UseDefaults() @@ -190,6 +209,17 @@ await ActivatorUtilities.CreateInstance(provider, storageOption .MigrateAsync(cancellationToken); } + static async Task CleanupAsync( + InvocationContext context, + CleanupOptions cleanupOptions, + CancellationToken cancellationToken) + { + var provider = context.BindingContext.GetRequiredService(); + await ActivatorUtilities.CreateInstance(provider, cleanupOptions) + .MigrateAsync(cancellationToken); + } + + static async Task MigrateKeysAsync( InvocationContext context, KeyOptions keyOptions, diff --git a/ams/CleanupCommand.cs b/ams/CleanupCommand.cs new file mode 100644 index 0000000..1c6560e --- /dev/null +++ b/ams/CleanupCommand.cs @@ -0,0 +1,198 @@ +using AMSMigrate.Ams; +using AMSMigrate.Contracts; +using Azure; +using Azure.Core; +using Azure.ResourceManager.Media; +using Azure.Storage.Blobs; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using System.ComponentModel; + +namespace AMSMigrate.ams +{ + internal class CleanupCommand : BaseMigrator + { + private readonly ILogger _logger; + private readonly CleanupOptions _options; + private readonly IMigrationTracker _tracker; + + public CleanupCommand(GlobalOptions globalOptions, + CleanupOptions cleanupOptions, + IAnsiConsole console, + TokenCredential credential, + IMigrationTracker tracker, + ILogger logger) + : base(globalOptions, console, credential) + { + _options = cleanupOptions; + _logger = logger; + _tracker = tracker; + } + public override async Task MigrateAsync(CancellationToken cancellationToken) + { + var account = await GetMediaAccountAsync(_options.AccountName, cancellationToken); + _logger.LogInformation("Begin cleaning up on account: {name}", account.Data.Name); + + if (_options.IsCleanUpAccount) + { + Console.Write($"Do you want to delete the account '{account.Data.Name}'? (y/n): "); + string? userResponse = Console.ReadLine(); + + if (!(userResponse?.ToLower() == "y")) + { + Console.WriteLine("Account cleanup canceled by user."); + return; + } + } + + Dictionary stats = new Dictionary(); + var totalAssets = await QueryMetricAsync( + account.Id.ToString(), + "AssetCount", + cancellationToken: cancellationToken); + + _logger.LogInformation("The total asset count of the media account is {count}.", totalAssets); + AsyncPageable assets; + + //clean up asset + var resourceFilter = _options.IsCleanUpAccount? null: GetAssetResourceFilter(_options.ResourceFilter, null, null); + + var orderBy = "properties/created"; + assets = account.GetMediaAssets() + .GetAllAsync(resourceFilter, orderby: orderBy, cancellationToken: cancellationToken); + List? assetList = await assets.ToListAsync(cancellationToken); + + foreach (var asset in assetList) + { + var result = await CleanUpAssetAsync(_options.IsCleanUpAccount||_options.IsForceCleanUpAsset,account, asset, cancellationToken); + stats.Add(asset.Data.Name, result); + } + WriteSummary(stats, false); + + if (_options.IsCleanUpAccount) + { + Dictionary accStats = new Dictionary(); + var result = await CleanUpAccountAsync(account, cancellationToken); + accStats.Add(account.Data.Name, result); + WriteSummary(accStats, true); + } + + } + + private void WriteSummary(IDictionary stats, bool isDeletingAccount) + { + var table = new Table(); + if (isDeletingAccount) + { + table.AddColumn("Account"); + } + else + { + table.AddColumn("Asset"); + } + table.AddColumn("IsDeleted"); + foreach (var (key, value) in stats) + { + var status = value ? $"[green]{value}[/]" : $"[red]{value}[/]"; + table.AddRow($"[green]{key}[/]", status); + } + + _console.Write(table); + } + private async Task CleanUpAccountAsync(MediaServicesAccountResource account, CancellationToken cancellationToken) + { + try + { + var endpoints = account.GetStreamingEndpoints(); + var liveevents = account.GetMediaLiveEvents(); + var policies = account.GetContentKeyPolicies(); + + if (endpoints != null) + { + foreach (var streamingEndpoint in endpoints) + { + await streamingEndpoint.DeleteAsync(WaitUntil.Completed); + } + } + if (policies != null) + { + foreach (var contentKeyPolicy in policies) + { + await contentKeyPolicy.DeleteAsync(WaitUntil.Completed); + } + } + if (liveevents != null) + { + foreach (var liveEvent in liveevents) + { + await liveEvent.DeleteAsync(WaitUntil.Completed); + } + } + + var deleteOperation = await account.DeleteAsync(WaitUntil.Completed); + + if (deleteOperation.HasCompleted && deleteOperation.GetRawResponse().Status == 200) + { + _logger.LogInformation("The media account {account} has been deleted.", account.Data.Name); + return true; + } + else + { + _logger.LogInformation("The media account {account} deletion failed.", account.Data.Name); + return false; + } + + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete account {name}", account.Data.Name); + return false; + } + } + private async Task CleanUpAssetAsync(bool isForcedelete,MediaServicesAccountResource account, MediaAssetResource asset, CancellationToken cancellationToken) + { + try + { + + var storage = await _resourceProvider.GetStorageAccountAsync(account, cancellationToken); + var container = storage.GetContainer(asset); + if (!await container.ExistsAsync(cancellationToken)) + { + _logger.LogWarning("Container {name} missing for asset {asset}", container.Name, asset.Data.Name); + + return false; + } + + // The asset container exists, try to check the metadata list first. + + if (isForcedelete||(_tracker.GetMigrationStatusAsync(container, cancellationToken).Result.Status == MigrationStatus.Completed)) + { + + var locator = await account.GetStreamingLocatorAsync(asset, cancellationToken); + if (locator != null) + { + await locator.DeleteAsync(WaitUntil.Completed); + } + + if (asset != null) + { + await asset.DeleteAsync(WaitUntil.Completed); + } + await container.DeleteAsync(); + _logger.LogDebug("locator: {locator}, Migrated asset: {asset} , container: {container} are deleted.", locator?.Data.Name, asset.Data.Name, container?.Name); + return true; + } + else + { + _logger.LogDebug("asset: {asset} does not meet the criteria for deletion.", asset.Data.Name); + return false; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete asset {name}", asset.Data.Name); + return false; + } + } + } +} diff --git a/contracts/CleanupOptions.cs b/contracts/CleanupOptions.cs new file mode 100644 index 0000000..ddf03bd --- /dev/null +++ b/contracts/CleanupOptions.cs @@ -0,0 +1,14 @@ +namespace AMSMigrate.Contracts +{ + /// + /// It holds the options for cleanup commands. + /// + public record CleanupOptions( + string AccountName, + string? ResourceFilter, + bool IsForceCleanUpAsset, + bool IsCleanUpAccount + ); + +} +