Skip to content

Add progress callback #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
869570967 wants to merge 7 commits into
base: main
Choose a base branch
from
+127 −52
Open

Add progress callback #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e5dc1be
1
Jun 15, 2026
722e217
1
869570967 Jun 15, 2026
6f47e1d
1
Jun 16, 2026
9df9199
1
869570967 Jun 16, 2026
36e3ac2
Add progress callback
Jun 22, 2026
7eae6b0
Merge branch 'main' of https://github.com/869570967/Python.Included
Jun 22, 2026
e3190f8
Remove some unnecessary comments, change the comments to Chinese, and…
Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Python.Deployment/DownloadInstallationSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public class DownloadInstallationSource : InstallationSource
/// </summary>
public string DownloadUrl { get; set; }

public override async Task<string> RetrievePythonZip(string destinationDirectory)
public override async Task<string> RetrievePythonZip(string destinationDirectory, Action<float> progress = null, CancellationToken token = default)
{
var zipFile = Path.Combine(destinationDirectory, GetPythonZipFileName());
if (!Force && File.Exists(zipFile))
Expand All @@ -53,7 +53,7 @@ public override async Task<string> RetrievePythonZip(string destinationDirectory
try
{
Log("Downloading source...");
await Downloader.Download(DownloadUrl, zipFile, progress => Log($"{progress:F2}%")).ConfigureAwait(false);
await Downloader.Download(DownloadUrl, zipFile, p => { Log($"{p:F2}%"); progress?.Invoke(p); }, token).ConfigureAwait(false);
Log("Done!");
return zipFile;
}
Expand Down
3 changes: 2 additions & 1 deletion Python.Deployment/EmbeddedResourceInstallationSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace Python.Deployment
Expand All @@ -51,7 +52,7 @@ public class EmbeddedResourceInstallationSource : InstallationSource
/// </summary>
public string ResourceName { get; set; }

public override Task<string> RetrievePythonZip(string destinationDirectory)
public override Task<string> RetrievePythonZip(string destinationDirectory, Action<float> progress = null, CancellationToken token = default)
{
var filePath = Path.Combine(destinationDirectory, ResourceName);
if (!Force && File.Exists(filePath))
Expand Down
3 changes: 2 additions & 1 deletion Python.Deployment/InstallationSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace Python.Deployment
Expand All @@ -46,7 +47,7 @@ public abstract class InstallationSource
/// </summary>
/// <param name="destinationDirectory">The directory location where the retrieved zip file should be placed</param>
/// <returns></returns>
public abstract Task<string> RetrievePythonZip(string destinationDirectory);
public abstract Task<string> RetrievePythonZip(string destinationDirectory, Action<float> progress = null, CancellationToken token = default);

/// <summary>
/// If true, retrieve the python file again even if it already exists at the destination path
Expand Down
153 changes: 113 additions & 40 deletions Python.Deployment/Installer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

Expand Down Expand Up @@ -73,12 +74,12 @@ private static void Log(string message)
LogMessage?.Invoke(message);
}

public static async Task SetupPython(bool force = false)
public static async Task SetupPython(Action<float> progress = null, CancellationToken token = default, bool force = false)
{
Environment.SetEnvironmentVariable("PATH", $"{EmbeddedPythonHome};" + Environment.GetEnvironmentVariable("PATH"));
if (!force && Directory.Exists(EmbeddedPythonHome) && File.Exists(Path.Combine(EmbeddedPythonHome, "python.exe"))) // python seems installed, so exit
return;
var zip = await Source.RetrievePythonZip(InstallPath).ConfigureAwait(false);
var zip = await Source.RetrievePythonZip(InstallPath, progress, token).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(zip))
{
Log("SetupPython: Error obtaining zip file from installation source");
Expand Down Expand Up @@ -242,7 +243,7 @@ public static async Task PipInstallWheel(Assembly assembly, string resource_name

CopyEmbeddedResourceToFile(assembly, key, wheelPath, force);

await TryInstallPip().ConfigureAwait(false);
await TryInstallPip(token: token).ConfigureAwait(false);

await RunCommand($"\"{pipPath}\" install \"{wheelPath}\"", token).ConfigureAwait(false);
}
Expand Down Expand Up @@ -289,19 +290,20 @@ public static string GetResourceKey(Assembly assembly, string embedded_file)
/// terminate when complete. When true, the command window must be manually closed before
/// processing will continue.
/// </param>
public static async Task PipInstallModule(string module_name, string version = "", bool force = false, CancellationToken token = default)
public static async Task PipInstallModule(string module_name, string version = "", bool force = false, Action<float> progress = null, CancellationToken token = default)
{
await TryInstallPip().ConfigureAwait(false);
await TryInstallPip(progress, token, force).ConfigureAwait(false);

if (IsModuleInstalled(module_name) && !force)
return;

string pipPath = Path.Combine(EmbeddedPythonHome, "Scripts", "pip");
string pythonPath = Path.Combine(EmbeddedPythonHome, "python.exe");
string forceInstall = force ? " --force-reinstall" : "";
if (version.Length > 0)
version = $"=={version}";

await RunCommand($"\"{pipPath}\" install \"{module_name}{version}\" {forceInstall}", token).ConfigureAwait(false);
await RunCommand(
$"-u -m pip install \"{module_name}{version}\" --no-cache-dir --progress-bar raw{forceInstall}", token, progress, pythonPath).ConfigureAwait(false);
}

/// <summary>
Expand All @@ -315,7 +317,7 @@ public static async Task PipInstallModule(string module_name, string version = "
/// terminate when complete. When true, the command window must be manually closed before
/// processing will continue.
/// </param>
public static async Task InstallPip(CancellationToken token = default)
public static async Task InstallPip(Action<float> progress = null, CancellationToken token = default)
{
string libDir = Path.Combine(EmbeddedPythonHome, "Lib");

Expand All @@ -328,7 +330,7 @@ public static async Task InstallPip(CancellationToken token = default)
try
{
Log("Downloading Pip...");
await Downloader.Download(getPipUrl, getPipFilePath, progress => Log($"{progress:F2}%")).ConfigureAwait(false);
await Downloader.Download(getPipUrl, getPipFilePath, p => { Log($"{p:F2}%"); progress?.Invoke(p); }).ConfigureAwait(false);
Log("Done!");
}
catch (Exception ex)
Expand All @@ -338,16 +340,16 @@ public static async Task InstallPip(CancellationToken token = default)
}


await RunCommand($"cd \"{EmbeddedPythonHome}\" && python.exe Lib\\get-pip.py", token).ConfigureAwait(false);
await RunCommand($"cd \"{EmbeddedPythonHome}\" && python.exe Lib\\get-pip.py", token, progress).ConfigureAwait(false);
}

public static async Task<bool> TryInstallPip(bool force = false)
public static async Task<bool> TryInstallPip(Action<float> progress = null, CancellationToken token = default, bool force = false)
{
if (!IsPipInstalled() || force)
{
try
{
await InstallPip().ConfigureAwait(false);
await InstallPip(progress, token).ConfigureAwait(false);
}
catch
{
Expand Down Expand Up @@ -377,43 +379,55 @@ public static bool IsModuleInstalled(string module)
return Directory.Exists(moduleDir) && File.Exists(Path.Combine(moduleDir, "__init__.py"));
}

public static async Task RunCommand(string command, CancellationToken token)

@henon henon Jun 24, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You didn't answer this question. Parameter filename was added but is never used. I would remove it because it is probably not possible to replace cmd.exe with something else without also replacing the args.

public static async Task RunCommand(string command, CancellationToken token, Action<float> progress = null, string filename = null)
{
Process process = new Process();
try
{
string args = null;
string filename = null;
ProcessStartInfo startInfo = new ProcessStartInfo();
string args;

if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// Unix/Linux/macOS specific command execution
filename = "/bin/bash";
args = $"-c \"{command} \"";
if (string.IsNullOrEmpty(filename))
filename = "/bin/bash";
args = $"-c \"{command}\"";
}
else
{
// Windows specific command execution
filename = "cmd.exe";
args = $"/C \"{command}\"";
if (string.IsNullOrEmpty(filename))
{
// Windows specific command execution
filename = "cmd.exe";
args = $"/C \"{command}\"";
}
else
{
args = command;
}
}

Log($"> {filename} {args}");
startInfo = new ProcessStartInfo

var startInfo = new ProcessStartInfo
Comment on lines +382 to +412

@henon henon Jun 24, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this necessary? What are you substituting as filename?

I guess if you use another shell then you may even need to customize the command line args as well. /C "..." probably only works only with cmd.exe

If in doubt, revert this.

{
FileName = filename,
WorkingDirectory = EmbeddedPythonHome,
Arguments = args,

// If the UseShellExecute property is true, the CreateNoWindow property value is ignored and a new window is created.
// .NET Core does not support creating windows directly on Unix/Linux/macOS and the property is ignored.

CreateNoWindow = true,
UseShellExecute = false, // necessary for stdout redirection
Comment on lines -406 to -411

@henon henon Jun 24, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do not remove existing comments, never change anything that is not relevant for the feature.

@869570967 869570967 Jun 24, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I’ve already made the corrections as you requested. Take another look.

Comment on lines -406 to -411

@henon henon Jun 24, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These commens still seem to be missing

@869570967 869570967 Jun 24, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some of the comments were given by AI, but I felt they were unnecessary, so I removed them. The reason for adding filename is that when installing Python libraries, you can't use the cmd command to install; instead, you have to set filename to the path of python.exe to start it. Only this way can you track the progress for callbacks. See line 305 for reference.

@869570967 869570967 Jun 24, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change mainly involves rewriting the RunCommand method. The other functions just added a parameter Action progress = null and passed it along layer by layer, so that the outermost function's Action progress = null can take effect.

@henon henon Jun 24, 2026 •
edited
Loading

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but as you can see from my screenshot, the original comments from before your change are still removed:

Image

You need to patch them back in.

UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
WindowStyle = ProcessWindowStyle.Hidden,
};

// Key: Disable output buffering for Python and pip
startInfo.Environment["PYTHONUNBUFFERED"] = "1";
startInfo.Environment["PYTHONIOENCODING"] = "utf-8";
startInfo.Environment["PIP_NO_COLOR"] = "1";
startInfo.Environment["COLUMNS"] = "200";

process.StartInfo = startInfo;
process.Start();
// Note: see https://github.com/henon/Python.Included/issues/55#issuecomment-1634750418
Expand All @@ -422,22 +436,33 @@ public static async Task RunCommand(string command, CancellationToken token)
//process.BeginErrorReadLine();
token.Register(() =>
{
try
{
if (!process.HasExited)
process.Kill();
}
try { if (!process.HasExited) process.Kill(); }
catch (Exception) { /* ignore */ }
});
// The documentation for Process.StandardOutput says to read before you wait otherwise you can deadlock!
string output = process.StandardOutput.ReadToEnd();
Log(output);
await Task.Run(() => { process.WaitForExit(); }, token).ConfigureAwait(false);
if (process.ExitCode != 0)

var readStdOut = ReadStreamAsync(process.StandardOutput, line =>
{
Log(process.StandardError.ReadToEnd());
Log(" => exit code " + process.ExitCode);
}
Log($"[ERR] '{line}'");
ParsePipProgress(line, progress);
}, token);

var readStdErr = ReadStreamAsync(process.StandardError, line =>
{
Log($"[ERR] '{line}'");
Console.WriteLine(line);
}, token);

await Task.WhenAll(readStdOut, readStdErr).ConfigureAwait(false);
await Task.Run(() => process.WaitForExit(), token).ConfigureAwait(false);

if (process.ExitCode == 0)
progress?.Invoke(100.0f);

Log(" => exit code " + process.ExitCode);
}
catch (OperationCanceledException)
{
Log("RunCommand: Cancelled");
}
catch (Exception e)
{
Expand All @@ -449,6 +474,54 @@ public static async Task RunCommand(string command, CancellationToken token)
}
}

private static async Task ReadStreamAsync(StreamReader reader, Action<string> callback, CancellationToken token)
{
var sb = new System.Text.StringBuilder();
char[] buf = new char[1];

while (!reader.EndOfStream && !token.IsCancellationRequested)
{
int read = await reader.ReadAsync(buf, 0, 1).ConfigureAwait(false);
if (read == 0) break;

char c = buf[0];
if (c == '\n' || c == '\r')
{
if (sb.Length > 0)
{
callback(sb.ToString());
sb.Clear();
}
}
else
{
sb.Append(c);
}
}

if (sb.Length > 0)
callback(sb.ToString());
}

private static void ParsePipProgress(string line, Action<float> progress)
{
if (progress == null) return;

var match = System.Text.RegularExpressions.Regex.Match(
line, @"Progress (\d+) of (\d+)"
);

if (match.Success)
{
if (long.TryParse(match.Groups[1].Value, out long current) &&
long.TryParse(match.Groups[2].Value, out long total) &&
total > 0)
{
float percent = (float)current / total * 100f;
progress(Math.Min(Math.Max(percent, 0f), 99f));
}
}
}
private static bool AreAllFilesAlreadyPresent(ZipArchive zip, string lib)
{
var allFilesAllReadyPresent = true;
Expand Down
16 changes: 8 additions & 8 deletions Python.Included/Installer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ private static void Log(string message)
LogMessage?.Invoke(message);
}

public static async Task SetupPython(bool force = false)
public static async Task SetupPython(Action<float> progress = null, CancellationToken token = default, bool force = false)
{
if (!PythonEnv.DeployEmbeddedPython)
return;
Expand All @@ -75,7 +75,7 @@ public static async Task SetupPython(bool force = false)
Python.Deployment.Installer.Source = GetInstallationSource();
Python.Deployment.Installer.PythonDirectoryName = InstallDirectory;
Python.Deployment.Installer.InstallPath = InstallPath;
await Python.Deployment.Installer.SetupPython(force).ConfigureAwait(false);
await Python.Deployment.Installer.SetupPython(progress, token, force).ConfigureAwait(false);
}
finally
{
Expand Down Expand Up @@ -149,15 +149,15 @@ public static async Task PipInstallWheel(Assembly assembly, string resource_name
/// </summary>
/// <param name="module_name">The module/package to install </param>
/// <param name="force">When true, reinstall the packages even if it is already up-to-date.</param>
public static async Task PipInstallModule(string module_name, string version = "", bool force = false, CancellationToken token = default)
public static async Task PipInstallModule(string module_name, string version = "", bool force = false, Action<float> progress = null, CancellationToken token = default)
{
try
{
Python.Deployment.Installer.LogMessage += Log;
Python.Deployment.Installer.Source = GetInstallationSource();
Python.Deployment.Installer.PythonDirectoryName = InstallDirectory;
Python.Deployment.Installer.InstallPath = InstallPath;
await Python.Deployment.Installer.PipInstallModule(module_name, version, force, token).ConfigureAwait(false);
await Python.Deployment.Installer.PipInstallModule(module_name, version, force, progress, token).ConfigureAwait(false);
}
finally
{
Expand All @@ -171,29 +171,29 @@ public static async Task PipInstallModule(string module_name, string version = "
/// <remarks>
/// Creates the lib folder under <see cref="EmbeddedPythonHome"/> if it does not exist.
/// </remarks>
public static async Task InstallPip(CancellationToken token = default)
public static async Task InstallPip(Action<float> progress = null, CancellationToken token = default)
{
try
{
Python.Deployment.Installer.LogMessage += Log;
Python.Deployment.Installer.Source = GetInstallationSource();
Python.Deployment.Installer.PythonDirectoryName = InstallDirectory;
Python.Deployment.Installer.InstallPath = InstallPath;
await Python.Deployment.Installer.InstallPip(token).ConfigureAwait(false);
await Python.Deployment.Installer.InstallPip(progress, token).ConfigureAwait(false);
}
finally
{
Python.Deployment.Installer.LogMessage -= Log;
}
}

public static async Task<bool> TryInstallPip(bool force = false)
public static async Task<bool> TryInstallPip(Action<float> progress = null, CancellationToken token = default, bool force = false)
{
if (!IsPipInstalled() || force)
{
try
{
await InstallPip().ConfigureAwait(false);
await InstallPip(progress, token).ConfigureAwait(false);
}
catch
{
Expand Down