Skip to content

Commit cfb6fab

Browse files
mlonskmlonsk
authored
Added DL/Ol To Import script (#292)
1 parent 394b8a8 commit cfb6fab

2 files changed

Lines changed: 284 additions & 1 deletion

File tree

‎content/features/CSharpScripts/Advanced/script-convert-dlol-to-import.md‎

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
---
2+
uid: script-convert-dlol-to-import
3+
title: Convert Direct Lake on OneLake to import
4+
author: Morten Lønskov
5+
updated: 2025-06-25
6+
applies_to:
7+
versions:
8+
- version: 2.x
9+
- version: 3.x
10+
---
11+
# Convert Import to Direct Lake on OneLake
12+
13+
## Script Purpose
14+
15+
This script converts Direct Lake on OneLake (DL/OL) to Import mode tables. As laid out in the [Direct Lake guidance article](xref:direct-lake-guidance), we need to replace the partition(s) on such tables with a single [EntityPartition](https://learn.microsoft.com/en-us/dotnet/api/microsoft.analysisservices.tabular.entitypartitionsource?view=analysisservices-dotnet), which specifies the name and schema of the table/materialized view in the Fabric Lakehouse or Warehouse, while referencing a Shared Expression that uses the [`AzureStorage.DataLake`](https://learn.microsoft.com/en-us/powerquery-m/azurestorage-datalake) (OneLake) connector.
16+
17+
## Prerequisites
18+
19+
You will need the **SQL Endpoint** as well as the **Name** of your Fabric Warehouse or Lakehouse. Both can be found in the Fabric portal.
20+
21+
You will also need to know the **Schema** of the table/materialized view you wish to connect to. For Lakehouses the default is dbo.
22+
23+
24+
## Script
25+
26+
### Convert Import mode tables to Direct Lake on OneLake
27+
28+
```csharp
29+
// ===================================================================================
30+
// Convert Direct Lake on OneLake tables back to Import mode
31+
// ----------------------------------------
32+
// This scripts converts the selected or all tables from Direct Lake on OneLake to import
33+
// It adds a shared expression named SQLEndpoint and replaces the existing DatabaseQuery if it no longer needed
34+
// ===================================================================================
35+
using System;
36+
using System.Linq;
37+
using System.Collections.Generic;
38+
using System.Windows.Forms;
39+
using System.Drawing;
40+
41+
// -------------------------------------------------------------------
42+
// 1) Scope‐picker dialog
43+
// -------------------------------------------------------------------
44+
public class ScopeSelectionDialog : Form
45+
{
46+
public enum ScopeOption { OnlySelected, All, Cancel }
47+
public ScopeOption SelectedOption { get; private set; }
48+
49+
public ScopeSelectionDialog(int selectedCount, int totalCount)
50+
{
51+
Text = "Choose tables to convert";
52+
AutoSize = true; AutoSizeMode = AutoSizeMode.GrowAndShrink;
53+
StartPosition = FormStartPosition.CenterParent;
54+
Padding = new Padding(20);
55+
56+
var layout = new TableLayoutPanel {
57+
ColumnCount = 1, Dock = DockStyle.Fill,
58+
AutoSize = true, AutoSizeMode = AutoSizeMode.GrowAndShrink
59+
};
60+
Controls.Add(layout);
61+
62+
layout.Controls.Add(new Label {
63+
Text = $"You have {selectedCount} table(s) selected,\nand {totalCount} Direct Lake table(s) in the model.",
64+
AutoSize = true, TextAlign = ContentAlignment.MiddleLeft
65+
});
66+
67+
var panel = new FlowLayoutPanel {
68+
FlowDirection = FlowDirection.LeftToRight,
69+
Dock = DockStyle.Fill, AutoSize = true,
70+
Padding = new Padding(0, 20, 0, 0)
71+
};
72+
73+
var btnOnly = new Button {
74+
Text = "Only selected tables", AutoSize = true,
75+
DialogResult = DialogResult.OK
76+
};
77+
btnOnly.Click += (s, e) => SelectedOption = ScopeOption.OnlySelected;
78+
79+
var btnAll = new Button {
80+
Text = "All tables", AutoSize = true,
81+
DialogResult = DialogResult.Retry
82+
};
83+
btnAll.Click += (s, e) => SelectedOption = ScopeOption.All;
84+
85+
var btnCancel = new Button {
86+
Text = "Cancel", AutoSize = true,
87+
DialogResult = DialogResult.Cancel
88+
};
89+
btnCancel.Click += (s, e) => SelectedOption = ScopeOption.Cancel;
90+
91+
panel.Controls.AddRange(new Control[] { btnOnly, btnAll, btnCancel });
92+
layout.Controls.Add(panel);
93+
94+
AcceptButton = btnOnly;
95+
CancelButton = btnCancel;
96+
}
97+
}
98+
99+
// -------------------------------------------------------------------
100+
// 2) SQL‐import dialog (schema now required)
101+
// -------------------------------------------------------------------
102+
public class SqlImportDialog : Form
103+
{
104+
public TextBox SqlEndpoint { get; }
105+
public TextBox DatabaseName { get; }
106+
public TextBox Schema { get; }
107+
private Button okButton;
108+
109+
public SqlImportDialog(string endpoint, string db, string schema)
110+
{
111+
Text = "Convert Direct Lake → Import";
112+
AutoSize = true; AutoSizeMode = AutoSizeMode.GrowAndShrink;
113+
StartPosition = FormStartPosition.CenterParent;
114+
Padding = new Padding(20);
115+
116+
var layout = new TableLayoutPanel {
117+
ColumnCount = 1, Dock = DockStyle.Fill,
118+
AutoSize = true, AutoSizeMode = AutoSizeMode.GrowAndShrink
119+
};
120+
Controls.Add(layout);
121+
122+
// Endpoint
123+
layout.Controls.Add(new Label { Text = "SQL Analytics Endpoint:", AutoSize = true });
124+
SqlEndpoint = new TextBox { Width = 800, Text = endpoint };
125+
layout.Controls.Add(SqlEndpoint);
126+
127+
// Database
128+
layout.Controls.Add(new Label {
129+
Text = "Lakehouse/Warehouse Name:", Padding = new Padding(0, 20, 0, 0),
130+
AutoSize = true
131+
});
132+
DatabaseName = new TextBox { Width = 800, Text = db };
133+
layout.Controls.Add(DatabaseName);
134+
135+
// Schema (required)
136+
layout.Controls.Add(new Label {
137+
Text = "Schema:", Padding = new Padding(0, 20, 0, 0),
138+
AutoSize = true
139+
});
140+
Schema = new TextBox { Width = 800, Text = schema };
141+
layout.Controls.Add(Schema);
142+
143+
// Buttons
144+
var panel = new FlowLayoutPanel {
145+
FlowDirection = FlowDirection.RightToLeft,
146+
Dock = DockStyle.Fill, AutoSize = true,
147+
Padding = new Padding(0, 20, 0, 0)
148+
};
149+
okButton = new Button {
150+
Text = "OK", DialogResult = DialogResult.OK,
151+
AutoSize = true, Enabled = false
152+
};
153+
var cancel = new Button {
154+
Text = "Cancel", DialogResult = DialogResult.Cancel,
155+
AutoSize = true
156+
};
157+
panel.Controls.AddRange(new Control[] { okButton, cancel });
158+
layout.Controls.Add(panel);
159+
160+
AcceptButton = okButton;
161+
CancelButton = cancel;
162+
163+
// Only enable OK when all three fields are non-empty
164+
SqlEndpoint.TextChanged += Validate;
165+
DatabaseName.TextChanged += Validate;
166+
Schema.TextChanged += Validate;
167+
Shown += (s,e) => Validate(s,e);
168+
}
169+
170+
private void Validate(object sender, EventArgs e)
171+
{
172+
okButton.Enabled =
173+
!string.IsNullOrWhiteSpace(SqlEndpoint.Text) &&
174+
!string.IsNullOrWhiteSpace(DatabaseName.Text) &&
175+
!string.IsNullOrWhiteSpace(Schema.Text);
176+
}
177+
}
178+
179+
// -------------------------------------------------------------------
180+
// 3) Main conversion logic
181+
// -------------------------------------------------------------------
182+
WaitFormVisible = false;
183+
Application.UseWaitCursor = false;
184+
185+
// 3.1) Find all Direct Lake tables
186+
var allDirectLake = Model.Tables
187+
.Where(t => t.Partitions.Count == 1
188+
&& t.Partitions[0].SourceType == PartitionSourceType.Entity
189+
&& t.Partitions[0].Mode == ModeType.DirectLake)
190+
.ToList();
191+
192+
// 3.2) And those you’ve selected
193+
var selectedDirect = Selected.Tables
194+
.Cast<Table>()
195+
.Where(t => t.Partitions.Count == 1
196+
&& t.Partitions[0].SourceType == PartitionSourceType.Entity
197+
&& t.Partitions[0].Mode == ModeType.DirectLake)
198+
.ToList();
199+
200+
// 3.3) Ask scope
201+
var scopeDialog = new ScopeSelectionDialog(selectedDirect.Count, allDirectLake.Count);
202+
var dr = scopeDialog.ShowDialog();
203+
if (dr == DialogResult.Cancel || scopeDialog.SelectedOption == ScopeSelectionDialog.ScopeOption.Cancel)
204+
return;
205+
206+
bool isAllTables = scopeDialog.SelectedOption == ScopeSelectionDialog.ScopeOption.All;
207+
var tablesToConvert = isAllTables
208+
? allDirectLake
209+
: selectedDirect;
210+
211+
if (tablesToConvert.Count == 0)
212+
{
213+
Warning("No Direct Lake tables found in the chosen scope.");
214+
return;
215+
}
216+
217+
// 3.4) Ask for connection + schema
218+
var sqlDialog = new SqlImportDialog("", "", "");
219+
if (sqlDialog.ShowDialog() == DialogResult.Cancel) return;
220+
221+
// 3.5) Upsert shared expression "SQLEndpoint"
222+
const string sqlTemplate = @"let
223+
endpoint = Sql.Database(""{0}"",""{1}"")
224+
in
225+
endpoint";
226+
var sqlexpr = Model.Expressions.FirstOrDefault(e => e.Name == "SQLEndpoint")
227+
?? Model.AddExpression("SQLEndpoint");
228+
sqlexpr.Expression = string.Format(
229+
sqlTemplate,
230+
sqlDialog.SqlEndpoint.Text,
231+
sqlDialog.DatabaseName.Text);
232+
233+
// 3.6) M‐partition template
234+
const string mTemplate = @"let
235+
Source = SQLEndpoint,
236+
Data = Source{{[Schema=""{0}"",Item=""{1}""]}}[Data]
237+
in
238+
Data";
239+
240+
// 3.7) Swap partitions
241+
foreach (var table in tablesToConvert)
242+
{
243+
var oldP = table.Partitions[0];
244+
oldP.Name += "_old";
245+
246+
var newP = table.AddMPartition(
247+
oldP.Name.Replace("_old", ""),
248+
string.Format(mTemplate, sqlDialog.Schema.Text, table.Name));
249+
newP.Mode = ModeType.Import;
250+
251+
oldP.Delete();
252+
}
253+
254+
// 3.8) If converting the **entire model**, delete the old DatabaseQuery expr
255+
if (isAllTables)
256+
{
257+
var oldDbq = Model.Expressions.FirstOrDefault(e => e.Name == "DatabaseQuery");
258+
if (oldDbq != null)
259+
oldDbq.Delete(); // TE3 API: Expression.Delete() removes it from the model
260+
}
261+
262+
// 3.9) Ensure default mode is Import
263+
Model.DefaultMode = ModeType.Import;
264+
265+
Info("Conversion complete: Direct Lake → Import" +
266+
(isAllTables ? " (DatabaseQuery removed)" : "") + ".");
267+
```
268+
269+
### Explanation
270+
271+
The script first prompts the user to determine the scope of the conversion by choosing between converting only the selected tables or all tables in the model. It then identifies which tables are currently in Direct Lake mode within the chosen scope. If no applicable tables are found, or if the user cancels the dialog, the script terminates.
272+
273+
The user is then prompted to enter the SQL Analytics Endpoint, the name of the Lakehouse or Warehouse, and a required Schema name. The script ensures all three fields are populated before allowing the user to proceed.
274+
275+
Next, the script creates or updates a Shared Expression named `SQLEndpoint`, using the provided connection details. This expression uses the `Sql.Database` connector to access the Lakehouse or Warehouse.
276+
277+
For each table being converted, the script creates a new Import mode M partition that references the `SQLEndpoint` expression and uses the specified schema and table name. The existing Direct Lake partition is renamed and then removed, leaving only the new Import partition.
278+
279+
Finally, if the user chose to convert all Direct Lake tables in the model, the script checks for an existing Shared Expression named `DatabaseQuery` and deletes it if found. The model's default storage mode is then set to Import, and a confirmation message is displayed.
280+
281+
## Use of AI disclaimer
282+
This script was created with the help of an LLM.

‎content/features/CSharpScripts/csharp-script-library-advanced.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,6 @@ These are more advanced scripts with sophisticated functionalities requiring a m
3232
| [Create Databricks Relationships](xref:script-create-databricks-relationships) | Create relationships based on primary and foreign key definitions in Databricks Unity Catalog | When you want to re-use Databricks relationship definitions that have already been defined in Unity Catalog. |
3333
| [Add Databricks Metadata Descriptions](xref:script-add-databricks-metadata-descriptions) | Update table and column descriptions based on Databricks Unity Catalog | When you want to re-use Databricks table and column comments that have already been defined in Unity Catalog. |
3434
| [Convert DL/SQL to DL/OL](xref:script-convert-dlsql-to-dlol) | Changes the partitions of a Direct Lake over SQL model to Direct Lake over OneLake | Useful for easily migrating to Direct Lake over OneLake |
35-
| [Convert Import to DL/OL](xref:script-convert-dlsql-to-dlol) | Changes the partitions of a Import model to Direct Lake over OneLake | Useful for easily migrating to Direct Lake over OneLake |
35+
| [Convert Import to DL/OL](xref:script-convert-import-to-dlol) | Changes the partitions of a Import model to Direct Lake over OneLake | Useful for easily migrating to Direct Lake over OneLake |
36+
| [Convert DL/OL to Import](xref:script-convert-dlol-to-import) | Changes the partitions of a Direct Lake over OneLake model to Import mode | Useful for easily migrating from Direct Lake over OneLake to Import |
3637
| [Implement User-defined Aggregations](xref:script-implement-user-defined-aggregations) | Automates the configuration of user-defined aggregations for a selected fact table. | When you want to implement the user-defined aggregations pattern without manually performing each configuration step. |

0 commit comments

Comments
 (0)