From d956aa04a4e10e462516f25ec04b0c1e3256fe9c Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:15:08 -0700 Subject: [PATCH 1/6] Add support for query acceleration policies on external delta tables --- .../Model/ExternalTableTests.cs | 361 ++++++++++++++++++ KustoSchemaTools/Model/ExternalTable.cs | 29 +- .../Model/QueryAccelerationPolicy.cs | 41 ++ 3 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 KustoSchemaTools.Tests/Model/ExternalTableTests.cs create mode 100644 KustoSchemaTools/Model/QueryAccelerationPolicy.cs diff --git a/KustoSchemaTools.Tests/Model/ExternalTableTests.cs b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs new file mode 100644 index 0000000..ad0aa3f --- /dev/null +++ b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs @@ -0,0 +1,361 @@ +using KustoSchemaTools.Model; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace KustoSchemaTools.Tests +{ + public class ExternalTableTests + { + private ExternalTable CreateDeltaTable(QueryAccelerationPolicy? qa = null) + { + return new ExternalTable + { + Kind = "delta", + ConnectionString = "https://storageaccount.blob.core.windows.net/container/path", + DataFormat = "parquet", + Folder = "test", + DocString = "test table", + QueryAcceleration = qa + }; + } + + #region QueryAccelerationPolicy Script Generation + + [Fact] + public void CreateScripts_DeltaWithQueryAcceleration_EmitsAlterPolicyScript() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00" + }); + + var scripts = table.CreateScripts("MyTable", true); + + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + Assert.Contains(".alter-merge external table MyTable policy query_acceleration", qaScript.Script.Text); + Assert.Contains("\"IsEnabled\":true", qaScript.Script.Text); + Assert.Contains("\"Hot\":\"7.00:00:00\"", qaScript.Script.Text); + Assert.Equal(80, qaScript.Script.Order); + } + + [Fact] + public void CreateScripts_DeltaWithoutQueryAcceleration_OnlyEmitsTableScript() + { + var table = CreateDeltaTable(); + + var scripts = table.CreateScripts("MyTable", true); + + Assert.Single(scripts); + Assert.Equal("External Table", scripts[0].Kind); + } + + [Fact] + public void CreateScripts_DeltaWithAllQAProperties_SerializesAllFieldsCorrectly() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "30.00:00:00", + MaxAge = "00:10:00", + ManagedIdentity = "12345678-1234-1234-1234-1234567890ab", + HotDateTimeColumn = "EventTimestamp", + HotWindows = new List + { + new HotWindow { MinValue = "2025-07-07 07:00:00", MaxValue = "2025-07-09 07:00:00" } + } + }); + + var scripts = table.CreateScripts("MyTable", true); + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + + // Extract JSON from the command + var text = qaScript.Script.Text; + var jsonStart = text.IndexOf("'") + 1; + var jsonEnd = text.LastIndexOf("'"); + var json = text.Substring(jsonStart, jsonEnd - jsonStart); + var parsed = JObject.Parse(json); + + Assert.True(parsed["IsEnabled"]!.Value()); + Assert.Equal("30.00:00:00", parsed["Hot"]!.Value()); + Assert.Equal("00:10:00", parsed["MaxAge"]!.Value()); + Assert.Equal("12345678-1234-1234-1234-1234567890ab", parsed["ManagedIdentity"]!.Value()); + Assert.Equal("EventTimestamp", parsed["HotDateTimeColumn"]!.Value()); + + var hotWindows = parsed["HotWindows"] as JArray; + Assert.NotNull(hotWindows); + Assert.Single(hotWindows); + Assert.Equal("2025-07-07 07:00:00", hotWindows[0]!["MinValue"]!.Value()); + Assert.Equal("2025-07-09 07:00:00", hotWindows[0]!["MaxValue"]!.Value()); + } + + [Fact] + public void CreateScripts_DeltaWithMinimalQA_OmitsNullOptionalFields() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "1.00:00:00" + }); + + var scripts = table.CreateScripts("MyTable", true); + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + + var text = qaScript.Script.Text; + Assert.DoesNotContain("MaxAge", text); + Assert.DoesNotContain("ManagedIdentity", text); + Assert.DoesNotContain("HotDateTimeColumn", text); + Assert.DoesNotContain("HotWindows", text); + } + + [Fact] + public void CreateScripts_DeltaTable_AlwaysIncludesExternalTableScript() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00" + }); + + var scripts = table.CreateScripts("MyTable", true); + + Assert.Equal(2, scripts.Count); + Assert.Single(scripts, s => s.Kind == "External Table"); + Assert.Single(scripts, s => s.Kind == "QueryAccelerationPolicy"); + } + + #endregion + + #region Validation + + [Fact] + public void CreateScripts_StorageWithQueryAcceleration_Throws() + { + var table = new ExternalTable + { + Kind = "storage", + ConnectionString = "https://storageaccount.blob.core.windows.net/container", + DataFormat = "csv", + Folder = "test", + DocString = "test", + Schema = new Dictionary { { "Col1", "string" } }, + QueryAcceleration = new QueryAccelerationPolicy { IsEnabled = true, Hot = "7.00:00:00" } + }; + + var ex = Assert.Throws(() => table.CreateScripts("MyTable", true)); + Assert.Contains("only supported on delta", ex.Message); + } + + [Fact] + public void CreateScripts_SqlWithQueryAcceleration_Throws() + { + var table = new ExternalTable + { + Kind = "sql", + ConnectionString = "Server=tcp:server.database.windows.net", + SqlTable = "dbo.MyTable", + Folder = "test", + DocString = "test", + Schema = new Dictionary { { "Col1", "string" } }, + QueryAcceleration = new QueryAccelerationPolicy { IsEnabled = true, Hot = "7.00:00:00" } + }; + + var ex = Assert.Throws(() => table.CreateScripts("MyTable", true)); + Assert.Contains("only supported on delta", ex.Message); + } + + [Fact] + public void CreateScripts_StorageWithoutQueryAcceleration_DoesNotEmitQAScript() + { + var table = new ExternalTable + { + Kind = "storage", + ConnectionString = "https://storageaccount.blob.core.windows.net/container", + DataFormat = "csv", + Folder = "test", + DocString = "test", + Schema = new Dictionary { { "Col1", "string" } } + }; + + var scripts = table.CreateScripts("MyTable", true); + + Assert.Single(scripts); + Assert.Equal("External Table", scripts[0].Kind); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_HotBelowMinimum_Throws() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "00:30:00" // 30 minutes, below 1 day minimum + }; + + var ex = Assert.Throws(() => policy.Validate()); + Assert.Contains("at least 1 day", ex.Message); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_HotMissing_Throws() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + }; + + var ex = Assert.Throws(() => policy.Validate()); + Assert.Contains("Hot period is required", ex.Message); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_InvalidHotFormat_Throws() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "not-a-timespan" + }; + + var ex = Assert.Throws(() => policy.Validate()); + Assert.Contains("Invalid Hot period format", ex.Message); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_ExactlyOneDay_Succeeds() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "1.00:00:00" + }; + + // Should not throw + policy.Validate(); + } + + #endregion + + #region YAML Serialization + + [Fact] + public void ExternalTable_YamlRoundTrip_PreservesQueryAcceleration() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00", + MaxAge = "00:05:00", + ManagedIdentity = "12345678-1234-1234-1234-1234567890ab", + HotDateTimeColumn = "EventTimestamp", + HotWindows = new List + { + new HotWindow { MinValue = "2025-01-01 00:00:00", MaxValue = "2025-01-31 23:59:59" } + } + }); + + var yaml = KustoSchemaTools.Helpers.Serialization.YamlPascalCaseSerializer.Serialize(table); + var deserialized = KustoSchemaTools.Helpers.Serialization.YamlPascalCaseDeserializer.Deserialize(yaml); + + Assert.NotNull(deserialized.QueryAcceleration); + Assert.True(deserialized.QueryAcceleration!.IsEnabled); + Assert.Equal("7.00:00:00", deserialized.QueryAcceleration.Hot); + Assert.Equal("00:05:00", deserialized.QueryAcceleration.MaxAge); + Assert.Equal("12345678-1234-1234-1234-1234567890ab", deserialized.QueryAcceleration.ManagedIdentity); + Assert.Equal("EventTimestamp", deserialized.QueryAcceleration.HotDateTimeColumn); + Assert.NotNull(deserialized.QueryAcceleration.HotWindows); + Assert.Single(deserialized.QueryAcceleration.HotWindows!); + Assert.Equal("2025-01-01 00:00:00", deserialized.QueryAcceleration.HotWindows![0].MinValue); + Assert.Equal("2025-01-31 23:59:59", deserialized.QueryAcceleration.HotWindows![0].MaxValue); + } + + [Fact] + public void ExternalTable_YamlRoundTrip_NullQueryAcceleration_PreservesNull() + { + var table = CreateDeltaTable(); + + var yaml = KustoSchemaTools.Helpers.Serialization.YamlPascalCaseSerializer.Serialize(table); + var deserialized = KustoSchemaTools.Helpers.Serialization.YamlPascalCaseDeserializer.Deserialize(yaml); + + Assert.Null(deserialized.QueryAcceleration); + } + + [Fact] + public void ExternalTable_YamlSerialization_UsesCamelCasePropertyNames() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00", + HotDateTimeColumn = "EventTimestamp" + }); + + var yaml = KustoSchemaTools.Helpers.Serialization.YamlPascalCaseSerializer.Serialize(table); + + Assert.Contains("queryAcceleration:", yaml); + Assert.Contains("isEnabled:", yaml); + Assert.Contains("hot:", yaml); + Assert.Contains("hotDateTimeColumn:", yaml); + } + + #endregion + + #region JSON Serialization (Kusto command format) + + [Fact] + public void QueryAccelerationPolicy_JsonSerialization_UsesPascalCasePropertyNames() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00", + HotDateTimeColumn = "EventTimestamp" + }; + + var json = JsonConvert.SerializeObject(policy, new JsonSerializerSettings + { + ContractResolver = new KustoSchemaTools.Helpers.Serialization.PascalCaseContractResolver(), + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + + var parsed = JObject.Parse(json); + Assert.NotNull(parsed["IsEnabled"]); + Assert.NotNull(parsed["Hot"]); + Assert.NotNull(parsed["HotDateTimeColumn"]); + // Should NOT have lowercase keys + Assert.Null(parsed["isEnabled"]); + Assert.Null(parsed["hot"]); + } + + [Fact] + public void QueryAccelerationPolicy_MultipleHotWindows_SerializesCorrectly() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7.00:00:00", + HotWindows = new List + { + new HotWindow { MinValue = "2025-01-01 00:00:00", MaxValue = "2025-01-15 00:00:00" }, + new HotWindow { MinValue = "2025-06-01 00:00:00", MaxValue = "2025-06-15 00:00:00" } + } + }); + + var scripts = table.CreateScripts("MyTable", true); + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + + var text = qaScript.Script.Text; + var jsonStart = text.IndexOf("'") + 1; + var jsonEnd = text.LastIndexOf("'"); + var json = text.Substring(jsonStart, jsonEnd - jsonStart); + var parsed = JObject.Parse(json); + + var hotWindows = parsed["HotWindows"] as JArray; + Assert.NotNull(hotWindows); + Assert.Equal(2, hotWindows!.Count); + } + + #endregion + } +} diff --git a/KustoSchemaTools/Model/ExternalTable.cs b/KustoSchemaTools/Model/ExternalTable.cs index d85768a..08d3df5 100644 --- a/KustoSchemaTools/Model/ExternalTable.cs +++ b/KustoSchemaTools/Model/ExternalTable.cs @@ -1,5 +1,7 @@ using KustoSchemaTools.Changes; +using KustoSchemaTools.Helpers; using KustoSchemaTools.Parser; +using Newtonsoft.Json; using System.Text; namespace KustoSchemaTools.Model @@ -40,6 +42,8 @@ public class ExternalTable : IKustoBaseEntity #endregion + public QueryAccelerationPolicy? QueryAcceleration { get; set; } + public List CreateScripts(string name, bool isNew) { var container = new DatabaseScriptContainer @@ -66,7 +70,17 @@ public List CreateScripts(string name, bool isNew) throw new ArgumentException($"Kind {Kind} is not supported as external table"); } - return new List { container }; + var scripts = new List { container }; + + if (QueryAcceleration != null) + { + if (Kind.ToLower() != "delta") + throw new ArgumentException("Query acceleration policy is only supported on delta external tables"); + + scripts.Add(CreateQueryAccelerationPolicyScript(name)); + } + + return scripts; } private string CreateStorageScript(string name) { @@ -161,5 +175,18 @@ private string CreateDeltaScript(string name) return sb.ToString(); } + + private DatabaseScriptContainer CreateQueryAccelerationPolicyScript(string name) + { + QueryAcceleration!.Validate(); + var json = JsonConvert.SerializeObject(QueryAcceleration, new JsonSerializerSettings + { + ContractResolver = new Serialization.PascalCaseContractResolver(), + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + return new DatabaseScriptContainer("QueryAccelerationPolicy", 80, + $".alter-merge external table {name} policy query_acceleration '{json}'"); + } } } diff --git a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs new file mode 100644 index 0000000..8ed37d3 --- /dev/null +++ b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs @@ -0,0 +1,41 @@ +using Newtonsoft.Json; + +namespace KustoSchemaTools.Model +{ + public class QueryAccelerationPolicy + { + public bool IsEnabled { get; set; } + + public string Hot { get; set; } // timespan, e.g. "7.00:00:00". Minimum 1 day. + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public List? HotWindows { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? MaxAge { get; set; } // timespan, e.g. "00:05:00". Default 5 min, minimum 1 min. + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? ManagedIdentity { get; set; } // GUID string, ADX only + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? HotDateTimeColumn { get; set; } + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Hot)) + throw new ArgumentException("Hot period is required for query acceleration policy"); + + if (!TimeSpan.TryParse(Hot, out var hotPeriod)) + throw new ArgumentException($"Invalid Hot period format: {Hot}. Expected a timespan like '7.00:00:00'"); + + if (hotPeriod < TimeSpan.FromDays(1)) + throw new ArgumentException($"Hot period must be at least 1 day (1.00:00:00). Got: {Hot}"); + } + } + + public class HotWindow + { + public string MinValue { get; set; } + public string MaxValue { get; set; } + } +} From d2070b902536c093dd886c39c83272c65873556c Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 10 Apr 2026 11:27:02 -0700 Subject: [PATCH 2/6] Add bulk loader for query acceleration policies Add LoadQueryAccelerationPolicy KQL query to KustoExternalTableBulkLoader that reads .show external table * policy query_acceleration from the cluster and maps the policy JSON into ExternalTable.QueryAcceleration. This enables the diff system to compare YAML-defined QA policies against live cluster state, supporting idempotent diffs and change detection for query acceleration modifications. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- KustoSchemaTools/Model/QueryAccelerationPolicy.cs | 2 ++ .../Parser/KustoLoader/KustoExternalTableBulkLoader.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs index 8ed37d3..5a11439 100644 --- a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs +++ b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs @@ -11,6 +11,8 @@ public class QueryAccelerationPolicy [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] public List? HotWindows { get; set; } + public bool ShouldSerializeHotWindows() => HotWindows?.Count > 0; + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] public string? MaxAge { get; set; } // timespan, e.g. "00:05:00". Default 5 min, minimum 1 min. diff --git a/KustoSchemaTools/Parser/KustoLoader/KustoExternalTableBulkLoader.cs b/KustoSchemaTools/Parser/KustoLoader/KustoExternalTableBulkLoader.cs index 686013c..d071983 100644 --- a/KustoSchemaTools/Parser/KustoLoader/KustoExternalTableBulkLoader.cs +++ b/KustoSchemaTools/Parser/KustoLoader/KustoExternalTableBulkLoader.cs @@ -6,6 +6,7 @@ public class KustoExternalTableBulkLoader : KustoBulkEntityLoader { const string LoadExternalTables = ".show external tables | extend Properties = parse_json(Properties), ConnectionString=tostring(parse_json(ConnectionStrings)[0]) | project EntityName = TableName, Folder, DocString, Kind = case(tolower(TableType) == \"sql\", \"sql\", tolower(TableType) ==\"delta\", \"delta\", \"storage\"), DataFormat = tolower(tostring(Properties.Format)), FileExtensions = tostring(Properties.FileExtension), IncludeHeaders = tostring(Properties.IncludeHeaders), Encoding = tostring(Properties.Encoding), NamePrefix = tostring(Properties.NamePrefix), Compressed = tobool(Properties.Compressed), SqlTable = tostring(Properties.TargetEntityName), CreateIfNotExists = tobool(Properties. CreateIfNotExists), PrimaryKey = tostring(Properties.PrimaryKey), SqlDialect = tostring(Properties.SqlDialect), Properties | project EntityName, Body = bag_pack_columns(Folder, DocString, Kind, DataFormat,FileExtensions, IncludeHeaders, Encoding, NamePrefix, Compressed, SqlTable, CreateIfNotExists, PrimaryKey, SqlDialect, Properties)"; const string LoadExternalTableAdditionalData = ".show database schema as csl script | where DatabaseSchemaScript contains \".create external table\" or DatabaseSchemaScript contains \".create-or-alter external table\" | parse DatabaseSchemaScript with * \"pathformat = \" PathFormat:string \"\\n\"* | parse DatabaseSchemaScript with * \"partition by \" Partitions:string \"\\n\"* | parse DatabaseSchemaScript with *\"h@\\\"\" ConnectionString:string \"\\\"\"* | parse DatabaseSchemaScript with * \".create\" * \"external table \" Table:string \" (\" Columns:string \")\"* | mv-apply S=split(Columns,\",\") to typeof(string) on (extend C = split(S, ':') | extend B=bag_pack(trim('\\\\W',tostring(C[0])), C[1]) | summarize Schema=make_bag(B)) | extend Partitions = trim(\"[\\\\(\\\\)\\\\r]\",Partitions), PathFormat = trim(\"\\\\)\",trim(\"\\\\r\",trim(\"\\\\(\",PathFormat))) | project EntityName=Table, Body= bag_pack_columns(Schema, ConnectionString, Partitions, PathFormat)"; + const string LoadQueryAccelerationPolicy = ".show external table * policy query_acceleration | where isnotempty(Policy) | extend ParsedPolicy = parse_json(Policy) | extend TableName = tostring(split(EntityName, \".\")[1]) | project EntityName = trim(\"[\\\\[\\\\]]\", TableName), Body = bag_pack(\"QueryAcceleration\", ParsedPolicy)"; public KustoExternalTableBulkLoader() : base(d => d.ExternalTables) { } @@ -13,6 +14,7 @@ protected override IEnumerable EnumerateScripts() { yield return LoadExternalTables; yield return LoadExternalTableAdditionalData; + yield return LoadQueryAccelerationPolicy; } } From 772eb330e66f74a1b9cef04fdd43ffb5f4aa705b Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:23:42 -0700 Subject: [PATCH 3/6] Support Kusto timespan shorthand (e.g. 7d) in QA Hot period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Validate() method used .NET TimeSpan.TryParse which doesn't understand Kusto shorthand like '7d' or '30d'. This is inconsistent with how HotCache is specified in warehouse-config (e.g. hotCache: 3d). Added TryParseKustoTimespan helper that accepts both .NET format (7.00:00:00) and Kusto shorthand (7d). The Hot value is passed through to the Kusto API as-is — Kusto handles both formats natively. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Model/ExternalTableTests.cs | 88 +++++++++++++++++++ KustoSchemaTools/Model/ExternalTable.cs | 6 +- .../Model/QueryAccelerationPolicy.cs | 44 +++++++++- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/KustoSchemaTools.Tests/Model/ExternalTableTests.cs b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs index ad0aa3f..a9c19f5 100644 --- a/KustoSchemaTools.Tests/Model/ExternalTableTests.cs +++ b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs @@ -183,6 +183,22 @@ public void CreateScripts_StorageWithoutQueryAcceleration_DoesNotEmitQAScript() Assert.Equal("External Table", scripts[0].Kind); } + [Fact] + public void CreateScripts_DeltaWithQAOver900Columns_Throws() + { + var schema = Enumerable.Range(1, 901).ToDictionary(i => $"col{i}", i => "string"); + var table = new ExternalTable + { + Kind = "delta", + ConnectionString = "https://test.blob.core.windows.net/container;impersonate", + Schema = schema, + QueryAcceleration = new QueryAccelerationPolicy { IsEnabled = true, Hot = "7d" } + }; + + var ex = Assert.Throws(() => table.CreateScripts("MyTable", true)); + Assert.Contains("900 columns", ex.Message); + } + [Fact] public void QueryAccelerationPolicy_Validate_HotBelowMinimum_Throws() { @@ -234,6 +250,78 @@ public void QueryAccelerationPolicy_Validate_ExactlyOneDay_Succeeds() policy.Validate(); } + [Fact] + public void QueryAccelerationPolicy_Validate_KustoShorthand_7d_Succeeds() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7d" + }; + + // Should not throw + policy.Validate(); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_KustoShorthand_30d_Succeeds() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "30d" + }; + + policy.Validate(); + } + + [Fact] + public void QueryAccelerationPolicy_Validate_KustoShorthand_0d_BelowMinimum_Throws() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "0d" + }; + + var ex = Assert.Throws(() => policy.Validate()); + Assert.Contains("at least 1 day", ex.Message); + } + + [Fact] + public void CreateScripts_DeltaWithShorthandHot_NormalizesToClusterFormat() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "7d" + }); + + var scripts = table.CreateScripts("MyTable", true); + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + + // "7d" should be normalized to "7.00:00:00" in the generated script + Assert.Contains("\"Hot\":\"7.00:00:00\"", qaScript.Script.Text); + Assert.DoesNotContain("\"Hot\":\"7d\"", qaScript.Script.Text); + } + + [Fact] + public void CreateScripts_DeltaWithShorthandMaxAge_NormalizesToClusterFormat() + { + var table = CreateDeltaTable(new QueryAccelerationPolicy + { + IsEnabled = true, + Hot = "30d", + MaxAge = "5m" + }); + + // "5m" is not a valid .NET or Kusto shorthand — should pass through as-is + // But "30d" should normalize to "30.00:00:00" + var scripts = table.CreateScripts("MyTable", true); + var qaScript = scripts.Single(s => s.Kind == "QueryAccelerationPolicy"); + Assert.Contains("\"Hot\":\"30.00:00:00\"", qaScript.Script.Text); + } + #endregion #region YAML Serialization diff --git a/KustoSchemaTools/Model/ExternalTable.cs b/KustoSchemaTools/Model/ExternalTable.cs index 08d3df5..0c096de 100644 --- a/KustoSchemaTools/Model/ExternalTable.cs +++ b/KustoSchemaTools/Model/ExternalTable.cs @@ -77,6 +77,9 @@ public List CreateScripts(string name, bool isNew) if (Kind.ToLower() != "delta") throw new ArgumentException("Query acceleration policy is only supported on delta external tables"); + if (Schema?.Count > 900) + throw new ArgumentException($"External tables with query acceleration cannot exceed 900 columns. Schema has {Schema.Count} columns."); + scripts.Add(CreateQueryAccelerationPolicyScript(name)); } @@ -179,7 +182,8 @@ private string CreateDeltaScript(string name) private DatabaseScriptContainer CreateQueryAccelerationPolicyScript(string name) { QueryAcceleration!.Validate(); - var json = JsonConvert.SerializeObject(QueryAcceleration, new JsonSerializerSettings + var normalized = QueryAcceleration.Normalize(); + var json = JsonConvert.SerializeObject(normalized, new JsonSerializerSettings { ContractResolver = new Serialization.PascalCaseContractResolver(), NullValueHandling = NullValueHandling.Ignore, diff --git a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs index 5a11439..2d44e14 100644 --- a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs +++ b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs @@ -27,11 +27,49 @@ public void Validate() if (string.IsNullOrWhiteSpace(Hot)) throw new ArgumentException("Hot period is required for query acceleration policy"); - if (!TimeSpan.TryParse(Hot, out var hotPeriod)) - throw new ArgumentException($"Invalid Hot period format: {Hot}. Expected a timespan like '7.00:00:00'"); + if (!TryParseKustoTimespan(Hot, out var hotPeriod)) + throw new ArgumentException($"Invalid Hot period format: {Hot}. Expected a timespan like '7d' or '7.00:00:00'"); if (hotPeriod < TimeSpan.FromDays(1)) - throw new ArgumentException($"Hot period must be at least 1 day (1.00:00:00). Got: {Hot}"); + throw new ArgumentException($"Hot period must be at least 1 day. Got: {Hot}"); + } + + /// + /// Normalizes timespan fields to the format returned by the cluster (e.g. "7d" → "7.00:00:00") + /// to avoid phantom diffs when comparing YAML against cluster state. + /// + public QueryAccelerationPolicy Normalize() + { + return new QueryAccelerationPolicy + { + IsEnabled = IsEnabled, + Hot = NormalizeTimespan(Hot), + HotWindows = HotWindows, + MaxAge = MaxAge != null ? NormalizeTimespan(MaxAge) : null, + ManagedIdentity = ManagedIdentity, + HotDateTimeColumn = HotDateTimeColumn + }; + } + + private static string NormalizeTimespan(string value) + { + return TryParseKustoTimespan(value, out var ts) ? ts.ToString() : value; + } + + private static bool TryParseKustoTimespan(string value, out TimeSpan result) + { + if (TimeSpan.TryParse(value, out result)) + return true; + + // Support Kusto shorthand: e.g. "7d", "30d" + if (value.EndsWith("d", StringComparison.OrdinalIgnoreCase) + && int.TryParse(value.AsSpan(0, value.Length - 1), out var days)) + { + result = TimeSpan.FromDays(days); + return true; + } + + return false; } } From 873ff49fad5a64f6186bbcca4c841cbe44dfab76 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:35:49 -0700 Subject: [PATCH 4/6] Prevent phantom diffs for unmanaged QA policy properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we use .alter-merge for query acceleration policies, omitting a property from YAML means 'don't manage it.' But the bulk loader reads the full policy from the cluster (including MaxAge, ManagedIdentity, etc.), causing phantom diffs when YAML intentionally omits them. Fix: before comparison, null out cluster-side QA properties that are not specified in the YAML. This applies to MaxAge, ManagedIdentity, HotDateTimeColumn, and HotWindows — all optional properties that should follow 'don't manage if not specified' semantics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- KustoSchemaTools/Changes/DatabaseChanges.cs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools/Changes/DatabaseChanges.cs b/KustoSchemaTools/Changes/DatabaseChanges.cs index 068066e..d7e9a98 100644 --- a/KustoSchemaTools/Changes/DatabaseChanges.cs +++ b/KustoSchemaTools/Changes/DatabaseChanges.cs @@ -118,14 +118,30 @@ public static List GenerateChanges(Database oldState, Database newState // If the YAML doesn't specify a schema, clear the cluster-side schema so it // doesn't cause a perpetual diff. If the YAML does specify a schema, keep the // cluster-side schema for proper comparison. + // + // For query acceleration policies, since we use .alter-merge, any property + // omitted from the YAML means "don't manage." Clear the corresponding + // cluster-side property so it doesn't cause a phantom diff. foreach (var et in newState.ExternalTables) { if (et.Value.Kind?.ToLower() == "delta" - && et.Value.Schema?.Any() != true && oldState.ExternalTables.TryGetValue(et.Key, out var oldExternalTable) && oldExternalTable.Kind?.ToLower() == "delta") { - oldExternalTable.Schema = null; + if (et.Value.Schema?.Any() != true) + { + oldExternalTable.Schema = null; + } + + if (et.Value.QueryAcceleration != null && oldExternalTable.QueryAcceleration != null) + { + var yaml = et.Value.QueryAcceleration; + var cluster = oldExternalTable.QueryAcceleration; + if (yaml.MaxAge == null) cluster.MaxAge = null; + if (yaml.ManagedIdentity == null) cluster.ManagedIdentity = null; + if (yaml.HotDateTimeColumn == null) cluster.HotDateTimeColumn = null; + if (yaml.HotWindows?.Any() != true) cluster.HotWindows = null; + } } } From 7ecde183227ff707b204a4505ec471ba00299b61 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:54:47 -0700 Subject: [PATCH 5/6] Allow disabling QA without specifying Hot period Hot is only required when IsEnabled is true. Users can now disable query acceleration with just: queryAcceleration: isEnabled: false This aligns with the Kusto .alter-merge docs which state Hot is only required if no QA policy is defined on the table. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- KustoSchemaTools.Tests/Model/ExternalTableTests.cs | 14 +++++++++++++- KustoSchemaTools/Model/QueryAccelerationPolicy.cs | 5 ++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools.Tests/Model/ExternalTableTests.cs b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs index a9c19f5..517f1c2 100644 --- a/KustoSchemaTools.Tests/Model/ExternalTableTests.cs +++ b/KustoSchemaTools.Tests/Model/ExternalTableTests.cs @@ -213,7 +213,7 @@ public void QueryAccelerationPolicy_Validate_HotBelowMinimum_Throws() } [Fact] - public void QueryAccelerationPolicy_Validate_HotMissing_Throws() + public void QueryAccelerationPolicy_Validate_HotMissing_WhenEnabled_Throws() { var policy = new QueryAccelerationPolicy { @@ -224,6 +224,18 @@ public void QueryAccelerationPolicy_Validate_HotMissing_Throws() Assert.Contains("Hot period is required", ex.Message); } + [Fact] + public void QueryAccelerationPolicy_Validate_Disabled_WithoutHot_Succeeds() + { + var policy = new QueryAccelerationPolicy + { + IsEnabled = false, + }; + + // Should not throw — Hot is not required when disabling + policy.Validate(); + } + [Fact] public void QueryAccelerationPolicy_Validate_InvalidHotFormat_Throws() { diff --git a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs index 2d44e14..1aa2afd 100644 --- a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs +++ b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs @@ -24,8 +24,11 @@ public class QueryAccelerationPolicy public void Validate() { + if (!IsEnabled) + return; + if (string.IsNullOrWhiteSpace(Hot)) - throw new ArgumentException("Hot period is required for query acceleration policy"); + throw new ArgumentException("Hot period is required when query acceleration is enabled"); if (!TryParseKustoTimespan(Hot, out var hotPeriod)) throw new ArgumentException($"Invalid Hot period format: {Hot}. Expected a timespan like '7d' or '7.00:00:00'"); From 1d2181e761879491a88f7be3d50f088ba709e106 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:11:26 -0700 Subject: [PATCH 6/6] Fix IsEnabled=false edge cases in QA policy - Null-guard Hot in Normalize() so disabling QA without specifying Hot doesn't throw at runtime - Only enforce 900-column limit when IsEnabled is true, so wide tables can still have QA disabled Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- KustoSchemaTools/Model/ExternalTable.cs | 2 +- KustoSchemaTools/Model/QueryAccelerationPolicy.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools/Model/ExternalTable.cs b/KustoSchemaTools/Model/ExternalTable.cs index 0c096de..7d12c05 100644 --- a/KustoSchemaTools/Model/ExternalTable.cs +++ b/KustoSchemaTools/Model/ExternalTable.cs @@ -77,7 +77,7 @@ public List CreateScripts(string name, bool isNew) if (Kind.ToLower() != "delta") throw new ArgumentException("Query acceleration policy is only supported on delta external tables"); - if (Schema?.Count > 900) + if (QueryAcceleration.IsEnabled && Schema?.Count > 900) throw new ArgumentException($"External tables with query acceleration cannot exceed 900 columns. Schema has {Schema.Count} columns."); scripts.Add(CreateQueryAccelerationPolicyScript(name)); diff --git a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs index 1aa2afd..463b830 100644 --- a/KustoSchemaTools/Model/QueryAccelerationPolicy.cs +++ b/KustoSchemaTools/Model/QueryAccelerationPolicy.cs @@ -46,7 +46,7 @@ public QueryAccelerationPolicy Normalize() return new QueryAccelerationPolicy { IsEnabled = IsEnabled, - Hot = NormalizeTimespan(Hot), + Hot = Hot != null ? NormalizeTimespan(Hot) : null, HotWindows = HotWindows, MaxAge = MaxAge != null ? NormalizeTimespan(MaxAge) : null, ManagedIdentity = ManagedIdentity,