diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/LMbench/latmemrd_example_results.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/LMbench/latmemrd_example_results.txt new file mode 100644 index 0000000000..0b1c6803f9 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/LMbench/latmemrd_example_results.txt @@ -0,0 +1,57 @@ +"stride=64 +0.00049 1.438 +0.00098 1.438 +0.00195 1.438 +0.00293 1.438 +0.00391 1.438 +0.00781 1.438 +0.01172 1.438 +0.01562 1.438 +0.03125 1.439 +0.04688 1.915 +0.06250 1.938 +0.12500 1.991 +0.18750 1.890 +0.25000 1.839 +0.50000 1.766 +0.75000 1.740 +1.00000 1.757 +2.00000 3.069 +3.00000 3.106 +4.00000 3.093 +8.00000 3.084 +12.00000 3.081 +16.00000 3.084 +32.00000 3.090 +48.00000 3.820 +64.00000 4.545 +128.00000 5.335 + +"stride=32 +0.00049 1.438 +0.00098 1.438 +0.00195 1.438 +0.00293 1.438 +0.00391 1.438 +0.00781 1.438 +0.01172 1.438 +0.01562 1.438 +0.03125 1.438 +0.04688 1.439 +0.06250 1.563 +0.12500 1.532 +0.18750 1.521 +0.25000 1.516 +0.50000 1.508 +0.75000 1.506 +1.00000 1.506 +2.00000 1.588 +3.00000 1.588 +4.00000 1.587 +8.00000 1.586 +12.00000 1.586 +16.00000 1.586 +32.00000 1.612 +48.00000 1.831 +64.00000 2.056 +128.00000 2.537 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/LMbench/LatMemRdMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/LMbench/LatMemRdMetricsParserTests.cs new file mode 100644 index 0000000000..3fdae8c733 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/LMbench/LatMemRdMetricsParserTests.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using System.Text; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class LatMemRdMetricsParserTests + { + private static string Examples = MockFixture.GetDirectory(typeof(LMbenchExecutorTests), "Examples", "LMbench"); + private MockFixture mockFixture; + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + } + + [Test] + public void LMbenchMetricsParserCapturesTheExpectedMetrics_1() + { + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string outputPath = this.mockFixture.Combine(LatMemRdMetricsParserTests.Examples, "latmemrd_example_results.txt"); + string results = File.ReadAllText(outputPath); + + LatMemRdMetricsParser parser = new LatMemRdMetricsParser(results); + IList metrics = parser.Parse(); + + Assert.IsTrue(metrics.Count == 54); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Latency_StrideBytes_64_Array_512_B" && m.Value == 1.438)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Latency_StrideBytes_64_Array_1_KiB" && m.Value == 1.438)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Latency_StrideBytes_32_Array_768_KiB" && m.Value == 1.506)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Latency_StrideBytes_32_Array_32_MiB" && m.Value == 1.612)); + } + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/LMbench/LMbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/LMbench/LMbenchExecutor.cs index b38d20e8f9..602ab3ae2c 100644 --- a/src/VirtualClient/VirtualClient.Actions/LMbench/LMbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/LMbench/LMbenchExecutor.cs @@ -5,6 +5,7 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Globalization; using System.IO.Abstractions; using System.Linq; @@ -12,6 +13,7 @@ namespace VirtualClient.Actions using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc.Razor; + using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.Extensions.DependencyInjection; using VirtualClient.Common; using VirtualClient.Common.Extensions; @@ -84,6 +86,28 @@ public IEnumerable Benchmarks } } + /// + /// Name of the Binary to run for the suite of Benchmarks compiled. + /// + public string BinaryName + { + get + { + return this.Parameters.GetValue(nameof(this.BinaryName), null); + } + } + + /// + /// Name of the Binary to run for the suite of Benchmarks compiled. + /// + public string BinaryCommandLine + { + get + { + return this.Parameters.GetValue(nameof(this.BinaryCommandLine), null); + } + } + /// /// The compilerFlags that are used for make command in compiling LMbench. /// @@ -119,7 +143,41 @@ public long? MemorySizeMB protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) { await this.BuildSourceCodeAsync(telemetryContext, cancellationToken); - await this.ExecuteWorkloadAsync(telemetryContext, cancellationToken); + if (this.BinaryName == null) + { + await this.ExecuteWorkloadAsync(telemetryContext, cancellationToken); + } + else if (this.BinaryName.Equals("lat_mem_rd", StringComparison.OrdinalIgnoreCase)) + { + string binaryPath = string.Empty; + if (this.Platform == PlatformID.Unix && this.CpuArchitecture == Architecture.X64) + { + binaryPath = this.PlatformSpecifics.Combine(this.LMbenchPackage.Path, "bin", "x86_64-Linux"); + } + else if (this.Platform == PlatformID.Unix && this.CpuArchitecture == Architecture.Arm64) + { + binaryPath = this.PlatformSpecifics.Combine(this.LMbenchPackage.Path, "bin", "aarch64-Linux"); + } + else + { + this.Logger.LogNotSupported(this.BinaryName, this.Platform, this.CpuArchitecture, EventContext.Persisted()); + } + + if (!string.IsNullOrEmpty(binaryPath)) + { + using (IProcessProxy executeBinary = await this.ExecuteCommandAsync(this.PlatformSpecifics.Combine(binaryPath, this.BinaryName), this.BinaryCommandLine, binaryPath, telemetryContext, cancellationToken)) + { + await this.LogProcessDetailsAsync(executeBinary, telemetryContext); + executeBinary.ThrowIfErrored(ProcessProxy.DefaultSuccessCodes, errorReason: ErrorReason.WorkloadFailed); + LatMemRdMetricsParser latMemRdMetricsParser = new LatMemRdMetricsParser($"{executeBinary.StandardOutput.ToString()}{executeBinary.StandardError.ToString()}"); + this.CaptureMetrics(executeBinary, latMemRdMetricsParser, telemetryContext, this.BinaryName); + } + } + } + else + { + this.Logger.LogMessage($"Unsupported {nameof(this.BinaryName)}: {this.BinaryName}, supported binarienames are : null(for sumarry),lat_rd_mem", telemetryContext); + } } /// @@ -180,7 +238,7 @@ private Task BuildSourceCodeAsync(EventContext telemetryContext, CancellationTok }); } - private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) + private void CaptureMetrics(IProcessProxy process, MetricsParser metricsParser, EventContext telemetryContext, string scenario) { this.MetadataContract.AddForScenario( "LMbench", @@ -189,12 +247,11 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext this.MetadataContract.Apply(telemetryContext); - LMbenchMetricsParser parser = new LMbenchMetricsParser(process.StandardOutput.ToString()); - IList metrics = parser.Parse(); + IList metrics = metricsParser.Parse(); this.Logger.LogMetrics( toolName: "LMbench", - scenarioName: "Memory Benchmark", + scenarioName: scenario, process.StartTime, process.ExitTime, metrics, @@ -268,7 +325,8 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok // The use of the original telemetry context created at the top // is purposeful. - this.CaptureMetrics(process, relatedContext); + LMbenchMetricsParser lmbenchMetricsParser = new LMbenchMetricsParser(process.StandardOutput.ToString()); + this.CaptureMetrics(process, lmbenchMetricsParser, relatedContext, "Memory Benchmark"); } }); } diff --git a/src/VirtualClient/VirtualClient.Actions/LMbench/LatMemRdMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/LMbench/LatMemRdMetricsParser.cs new file mode 100644 index 0000000000..3e0ad35c13 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/LMbench/LatMemRdMetricsParser.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using VirtualClient.Contracts; + +namespace VirtualClient.Actions +{ + /// + /// + /// + public class LatMemRdMetricsParser : MetricsParser + { + /// + /// Sectionize by one or more empty lines. + /// + private static readonly Regex LatMemRdSectionDelimiter = new Regex(@$"({Environment.NewLine})(\s)*({Environment.NewLine})", RegexOptions.ExplicitCapture); + + /// + /// Initializes a new instance of the class. + /// + /// + public LatMemRdMetricsParser(string rawText) + : base(rawText) + { + } + + /// + public override IList Parse() + { + IList metrics = new List(); + this.Preprocess(); + this.Sections = TextParsingExtensions.Sectionize(this.PreprocessedText, LatMemRdSectionDelimiter); + foreach (var section in this.Sections) + { + var lines = section.Value.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + foreach (var line in lines) + { + var values = line.Split(' '); + string strideSize = section.Key.Split('=')[1]; + var metadata = new Dictionary(); + metadata.Add("StrideSizeBytes", strideSize); + metadata.Add("ArraySizeInMiB", values[0]); + long arraySizeInBytes = this.RoundOffToNearest512Multiple(double.Parse(values[0]) * 1024 * 1024) * 512; + metrics.Add(new Metric($"Latency_StrideBytes_{strideSize}_Array_{this.MetricNameSuffix(arraySizeInBytes)}", double.Parse(values[1]), "ns", MetricRelativity.LowerIsBetter, null, $"Latency for memory read operation for Array size in MB {values[0]} & stride size {strideSize} in nano seconds", metadata)); + } + } + + return metrics; + + } + + /// + protected override void Preprocess() + { + // Removing unnecessary starting and ending space. + this.PreprocessedText = this.RawText.Trim(); + } + + private long RoundOffToNearest512Multiple(double number) + { + return (long)Math.Round(number / 512.0); + } + + private string MetricNameSuffix(double bytes) + { + if (bytes >= 1024 * 1024) + { + return $"{bytes / (1024 * 1024)}_MiB"; + } + else if (bytes >= 1024) + { + return $"{bytes / 1024}_KiB"; + } + else + { + return $"{bytes}_B"; + } + + } + } +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MEM-LATRDMEM.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MEM-LATRDMEM.json new file mode 100644 index 0000000000..406a055f32 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MEM-LATRDMEM.json @@ -0,0 +1,126 @@ +{ + "Description": "LMbench Performance Workload", + "MinimumExecutionInterval": "00:01:00", + "Metadata": { + "RecommendedMinimumExecutionTime": "(4-cores)=04:00:00,(16-cores)=10:00:00,(64-cores)=16:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu", + "Notes_Runtime": "The benchmark takes approximately 6 to 8 minutes per 1 GB of RAM targeted. The default profile uses 25% of the total RAM." + }, + "Parameters": { + "CompilerName": "gcc", + "CompilerVersion": "10", + "CompilerFlags": "CPPFLAGS=\"-I /usr/include/tirpc\"" + }, + "Actions": [ + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_8B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 8" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_16B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 16" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_32B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 32" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_64B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 64" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_128B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 128" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_256B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 256" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_512B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 512" + } + }, + { + "Type": "LMbenchExecutor", + "Parameters": { + "Scenario": "LatMemRd_Array_4096MB_Stride_1024B", + "PackageName": "lmbench", + "CompilerFlags": "$.Parameters.CompilerFlags", + "BinaryName": "lat_mem_rd", + "BinaryCommandLine": "4096 1024" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "libtirpc-dev", + "Packages-Yum": "libtirpc-devel", + "Packages-Dnf": "libtirpc-devel" + } + }, + { + "Type": "CompilerInstallation", + "Parameters": { + "Scenario": "InstallCompiler", + "CompilerName": "$.Parameters.CompilerName", + "CompilerVersion": "$.Parameters.CompilerVersion" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallLMbenchPackages", + "BlobContainer": "packages", + "BlobName": "lmbench.3.0-r1324.zip", + "PackageName": "lmbench", + "Extract": true + } + } + ] +} \ No newline at end of file