From 9a2ea7b550f7581f1be54c2d770aa31a88b2ad86 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 18 Aug 2022 18:28:00 -0700 Subject: [PATCH 01/52] unmanaged node prototype --- .../ApiService/Functions/AgentRegistration.cs | 62 ++++++++----------- .../ApiService/Functions/GetPoolConfig.cs | 58 +++++++++++++++++ .../ApiService/OneFuzzTypes/Model.cs | 7 ++- .../ApiService/OneFuzzTypes/Requests.cs | 14 +++++ .../ApiService/OneFuzzTypes/Responses.cs | 1 + src/ApiService/ApiService/UserCredentials.cs | 39 +++++++----- .../onefuzzlib/EndpointAuthorization.cs | 8 +++ .../ApiService/onefuzzlib/Extension.cs | 18 ++++-- src/ApiService/IntegrationTests/JobsTests.cs | 2 +- src/ApiService/IntegrationTests/NodeTests.cs | 10 +-- src/ApiService/IntegrationTests/PoolTests.cs | 10 +-- src/ApiService/Tests/RemoveUserInfoTest.cs | 3 +- src/agent/onefuzz-agent/src/config.rs | 5 +- src/cli/onefuzz/api.py | 11 ++++ src/deployment/deploy.py | 10 ++- src/deployment/deploylib/registration.py | 31 ++++++++-- 16 files changed, 214 insertions(+), 75 deletions(-) create mode 100644 src/ApiService/ApiService/Functions/GetPoolConfig.cs diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index 2daf72791e..40255c2390 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -1,5 +1,4 @@ -using System.Web; -using Azure.Storage.Sas; +using Azure.Storage.Sas; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; @@ -31,17 +30,13 @@ public Async.Task Run( }); private async Async.Task Get(HttpRequestData req) { - var uri = HttpUtility.ParseQueryString(req.Url.Query); - var rawMachineId = uri["machine_id"]; - if (rawMachineId is null || !Guid.TryParse(rawMachineId, out var machineId)) { - return await _context.RequestHandling.NotOk( - req, - new Error( - ErrorCode.INVALID_REQUEST, - new string[] { "'machine_id' query parameter must be provided" }), - "agent registration"); + var request = await RequestHandling.ParseUri(req); + if (!request.IsOk) { + return await _context.RequestHandling.NotOk(req, request.ErrorV, "agent registration"); } + var machineId = request.OkV.MachineId; + var agentNode = await _context.NodeOperations.GetByMachineId(machineId); if (agentNode is null) { return await _context.RequestHandling.NotOk( @@ -82,32 +77,18 @@ private async Async.Task CreateRegistrationResponse(S WorkQueue: workQueue); } + // todo: add agent registration post private async Async.Task Post(HttpRequestData req) { - var uri = HttpUtility.ParseQueryString(req.Url.Query); - var rawMachineId = uri["machine_id"]; - if (rawMachineId is null || !Guid.TryParse(rawMachineId, out var machineId)) { - return await _context.RequestHandling.NotOk( - req, - new Error( - ErrorCode.INVALID_REQUEST, - new string[] { "'machine_id' query parameter must be provided" }), - "agent registration"); - } - - var rawPoolName = uri["pool_name"]; - if (rawPoolName is null || !PoolName.TryParse(rawPoolName, out var poolName)) { - return await _context.RequestHandling.NotOk( - req, - new Error( - ErrorCode.INVALID_REQUEST, - new string[] { "'pool_name' query parameter must be provided" }), - "agent registration"); + var request = await RequestHandling.ParseUri(req); + if (!request.IsOk) { + return await _context.RequestHandling.NotOk(req, request.ErrorV, "agent registration"); } - var rawScalesetId = uri["scaleset_id"]; - var scalesetId = rawScalesetId is null ? (Guid?)null : Guid.Parse(rawScalesetId); - - var version = uri["version"] ?? "1.0.0"; + var machineId = request.OkV.MachineId; + var poolName = request.OkV.PoolName; + var scalesetId = request.OkV.ScalesetId; + var version = request.OkV.Version; + var os = request.OkV.Os; _log.Info($"registration request: machine_id: {machineId} pool_name: {poolName} scaleset_id: {scalesetId} version: {version}"); var poolResult = await _context.PoolOperations.GetByName(poolName); @@ -127,12 +108,23 @@ private async Async.Task Post(HttpRequestData req) { await _context.NodeOperations.Delete(existingNode); } + if (os != null && pool.Os != os) { + return await _context.RequestHandling.NotOk( + req, + new Error( + Code: ErrorCode.INVALID_REQUEST, + Errors: new[] { $"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'" }), + "agent registration"); + } + var node = new Service.Node( PoolName: poolName, PoolId: pool.PoolId, MachineId: machineId, ScalesetId: scalesetId, - Version: version); + Version: version, + Os: os ?? pool.Os + ); await _context.NodeOperations.Replace(node); diff --git a/src/ApiService/ApiService/Functions/GetPoolConfig.cs b/src/ApiService/ApiService/Functions/GetPoolConfig.cs new file mode 100644 index 0000000000..266a7dab6b --- /dev/null +++ b/src/ApiService/ApiService/Functions/GetPoolConfig.cs @@ -0,0 +1,58 @@ +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; + +namespace Microsoft.OneFuzz.Service.Functions; + +public class GetPoolConfig { + private readonly ILogTracer _log; + private readonly IEndpointAuthorization _auth; + private readonly IOnefuzzContext _context; + + public GetPoolConfig(ILogTracer log, IEndpointAuthorization auth, IOnefuzzContext context) { + _log = log; + _auth = auth; + _context = context; + } + + [Function("GetPoolConfig")] + public Async.Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", Route = "pool/getconfig")] HttpRequestData req, ClaimsPrincipal principal) + => _auth.CallIfUser(req, r => r.Method switch { + "GET" => Get(r), + var m => throw new InvalidOperationException("Unsupported HTTP method {m}"), + }); + + private async Task Get(HttpRequestData req) { + var request = await RequestHandling.ParseRequest(req); + if (!request.IsOk) { + return await _context.RequestHandling.NotOk(req, request.ErrorV, "pool get config"); + } + + var search = request.OkV; + OneFuzzResult poolResult; + if (search.PoolId is Guid poolId) { + poolResult = await _context.PoolOperations.GetById(poolId); + } else if (search.Name is PoolName name) { + poolResult = await _context.PoolOperations.GetByName(name); + } else { + return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "missing pool name or id" }), "pool get config"); + } + + if (!poolResult.IsOk) { + return await _context.RequestHandling.NotOk(req, poolResult.ErrorV, context: search.ToString()); + } + + if (!poolResult.OkV.Managed) { + return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "Pool is a managed pool" }), context: search.ToString()); + } + var poolConfig = await _context.Extensions.CreatePoolConfig(poolResult.OkV); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(poolConfig); + return response; + } + + +} diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index ac87e38dcb..610488f9f9 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -96,6 +96,7 @@ public record Node DateTimeOffset? Heartbeat = null, DateTimeOffset? InitializedAt = null, NodeState State = NodeState.Init, + Os? Os = null, Guid? ScalesetId = null, bool ReimageRequested = false, @@ -162,7 +163,11 @@ public sealed override string ToString() { } }; -public record UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn); +public record UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn, List Roles) { + public static UserInfo Create() { + return new UserInfo(null, null, null, new List()); + } +} public record TaskDetails( TaskType Type, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 4b939f2108..e830fb0f8a 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -252,3 +252,17 @@ public record WebhookUpdate( string? SecretToken, WebhookMessageFormat? MessageFormat ); + + +public record AgentRegistrationGet( + Guid MachineId +); + + +public record AgentRegistrationPost( + PoolName PoolName, + Guid? ScalesetId, + Guid MachineId, + string Version, + Os? Os +); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index f5cbfa9871..6dd1cea57f 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -103,6 +103,7 @@ public record PoolGetResult( AgentConfig? Config, List? WorkQueue, List? ScalesetSummary +//List? UnmanagedNodes ) : BaseResponse(); public record ScalesetResponse( diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index 4083d1527f..9023ba3202 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -1,4 +1,5 @@ -using System.Net.Http.Headers; +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.IdentityModel.Tokens; @@ -15,10 +16,12 @@ public interface IUserCredentials { public class UserCredentials : IUserCredentials { ILogTracer _log; IConfigOperations _instanceConfig; + private JwtSecurityTokenHandler _tokenHandler; public UserCredentials(ILogTracer log, IConfigOperations instanceConfig) { _log = log; _instanceConfig = instanceConfig; + _tokenHandler = new JwtSecurityTokenHandler(); } public string? GetBearerToken(HttpRequestData req) { @@ -57,6 +60,8 @@ from t in r.AllowedAadTenants } public virtual async Task> ParseJwtToken(HttpRequestData req) { + + var authToken = GetAuthToken(req); if (authToken is null) { return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find authorization token" }); @@ -65,22 +70,24 @@ public virtual async Task> ParseJwtToken(HttpRequestData var allowedTenants = await GetAllowedTenants(); if (allowedTenants.IsOk) { if (allowedTenants.OkV is not null && allowedTenants.OkV.Contains(token.Issuer)) { - Guid? applicationId = ( - from t in token.Claims - where t.Type == "appId" - select (Guid.Parse(t.Value))).FirstOrDefault(); - - Guid? objectId = ( - from t in token.Claims - where t.Type == "oid" - select (Guid.Parse(t.Value))).FirstOrDefault(); - - string? upn = ( - from t in token.Claims - where t.Type == "upn" - select t.Value).FirstOrDefault(); + var userInfo = + token.Payload.Claims.Aggregate(UserInfo.Create(), (acc, claim) => { + switch (claim.Type) { + case "oid": + return acc with { ObjectId = Guid.Parse(claim.Value) }; + case "appId": + return acc with { ApplicationId = Guid.Parse(claim.Value) }; + case "upn": + return acc with { Upn = claim.Value }; + case "roles": + acc.Roles.Add(claim.Value); + return acc; + default: + return acc; + } + }); - return OneFuzzResult.Ok(new(applicationId, objectId, upn)); + return OneFuzzResult.Ok(userInfo); } else { _log.Error($"issuer not from allowed tenant: {token.Issuer} - {allowedTenants}"); return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unauthorized AAD issuer" }); diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index cfdaad9b97..4bc8598c88 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -5,6 +5,7 @@ namespace Microsoft.OneFuzz.Service; public interface IEndpointAuthorization { + Async.Task CallIfAgent( HttpRequestData req, Func> method) @@ -183,6 +184,9 @@ private GroupMembershipChecker CreateGroupMembershipChecker(InstanceConfig confi } public async Async.Task IsAgent(UserInfo tokenData) { + // todo: handle unmanaged node here + // check if the request is comming from a geristered app with apprile agent + if (tokenData.ObjectId != null) { var scalesets = _context.ScalesetOperations.GetByObjectId(tokenData.ObjectId.Value); if (await scalesets.AnyAsync()) { @@ -202,6 +206,10 @@ public async Async.Task IsAgent(UserInfo tokenData) { return true; } + // if (tokenData.Roles.Contains("unmanagedNode")) { + // return true; + // } + return false; } } diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index 22f8c447f7..f905c1183b 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -12,6 +12,9 @@ public interface IExtensions { Async.Task> FuzzExtensions(Pool pool, Scaleset scaleset); Async.Task> ReproExtensions(AzureLocation region, Os reproOs, Guid reproId, ReproConfig reproConfig, Container? setupContainer); + + Async.Task BuildPoolConfig(Pool pool); + Task CreatePoolConfig(Pool pool); Task> ProxyManagerExtensions(string region, Guid proxyId); } @@ -211,6 +214,14 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) { public async Async.Task BuildPoolConfig(Pool pool) { + var config = await CreatePoolConfig(pool); + + var fileName = $"{pool.Name}/config.json"; + await _context.Containers.SaveBlob(new Container("vm-scripts"), fileName, (JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions())), StorageType.Config); + return await ConfigUrl(new Container("vm-scripts"), fileName, false); + } + + public async Task CreatePoolConfig(Pool pool) { var instanceId = await _context.Containers.GetInstanceId(); var queueSas = await _context.Queue.GetQueueSas("node-heartbeat", StorageType.Config, QueueSasPermissions.Add); @@ -223,11 +234,8 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) { MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, InstanceId: instanceId - ); - - var fileName = $"{pool.Name}/config.json"; - await _context.Containers.SaveBlob(new Container("vm-scripts"), fileName, (JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions())), StorageType.Config); - return await ConfigUrl(new Container("vm-scripts"), fileName, false); + ); + return config; } diff --git a/src/ApiService/IntegrationTests/JobsTests.cs b/src/ApiService/IntegrationTests/JobsTests.cs index b33469b947..0749a6d1f7 100644 --- a/src/ApiService/IntegrationTests/JobsTests.cs +++ b/src/ApiService/IntegrationTests/JobsTests.cs @@ -157,7 +157,7 @@ public async Async.Task Post_CreatesJob_AndContainer() { var func = new Jobs(auth, Context); // need user credentials to put into the job object - var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "upn"); + var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var result = await func.Run(TestHttpRequestData.FromJson("POST", _config)); diff --git a/src/ApiService/IntegrationTests/NodeTests.cs b/src/ApiService/IntegrationTests/NodeTests.cs index 69da450610..39d70ac053 100644 --- a/src/ApiService/IntegrationTests/NodeTests.cs +++ b/src/ApiService/IntegrationTests/NodeTests.cs @@ -161,7 +161,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -189,7 +189,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -219,7 +219,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -250,7 +250,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -279,7 +279,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); // all of these operations use NodeGet diff --git a/src/ApiService/IntegrationTests/PoolTests.cs b/src/ApiService/IntegrationTests/PoolTests.cs index c3405d66f3..f461b12da1 100644 --- a/src/ApiService/IntegrationTests/PoolTests.cs +++ b/src/ApiService/IntegrationTests/PoolTests.cs @@ -145,7 +145,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -167,7 +167,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Halt, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -189,7 +189,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -210,7 +210,7 @@ await Context.InsertAll( new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }); // needed for admin check // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); // need to override instance id @@ -241,7 +241,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); diff --git a/src/ApiService/Tests/RemoveUserInfoTest.cs b/src/ApiService/Tests/RemoveUserInfoTest.cs index 84116c475e..4641e6aca1 100644 --- a/src/ApiService/Tests/RemoveUserInfoTest.cs +++ b/src/ApiService/Tests/RemoveUserInfoTest.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text.Json; using Microsoft.OneFuzz.Service; using Xunit; @@ -8,7 +9,7 @@ public class RemoveUserInfoTest { [Fact] void TestSerilize() { - var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "test"); + var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "test", new List()); var options = new JsonSerializerOptions(); options.Converters.Add(new RemoveUserInfo()); var serialized = JsonSerializer.Serialize(userInfo, options); diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index b15129ad3d..60ee143b89 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -212,7 +212,10 @@ impl Registration { url.query_pairs_mut() .append_pair("machine_id", &machine_id.to_string()) .append_pair("pool_name", &config.pool_name) - .append_pair("version", env!("ONEFUZZ_VERSION")); + .append_pair("version", env!("ONEFUZZ_VERSION")) + .append_pair("os", std::env::consts::OS); + + //todo: add os version? if managed { let scaleset = onefuzz::machine_id::get_scaleset_name().await?; diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 0038855b6d..e45e9dc347 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1244,6 +1244,17 @@ def list( ) + def get_unamaged_config(self, name: str) -> models.AgentConfig: + self.logger.debug("get details on a specific pool") + expanded_name = self._disambiguate( + "pool name", name, lambda x: False, lambda: [x.name for x in self.list()] + ) + + return self._req_model( + "GET", models.AgentConfig, data=requests.PoolSearch(name=expanded_name), alternate_endpoint="pool/getconfig" + ) + + class Node(Endpoint): """Interact with nodes""" diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index ba7b85b8fb..6487ca1290 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -379,7 +379,7 @@ def setup_rbac(self) -> None: }, { "allowedMemberTypes": ["Application"], - "description": "Allow access from a lab machine.", + "description": "Allow access from a managed node.", "displayName": OnefuzzAppRole.ManagedNode.value, "id": str(uuid.uuid4()), "isEnabled": True, @@ -393,6 +393,14 @@ def setup_rbac(self) -> None: "isEnabled": True, "value": OnefuzzAppRole.UserAssignment.value, }, + { + "allowedMemberTypes": ["Application"], + "description": "Allow access from an unmanaged node.", + "displayName": OnefuzzAppRole.UnmanagedNode.value, + "id": str(uuid.uuid4()), + "isEnabled": True, + "value": OnefuzzAppRole.UnmanagedNode.value, + }, ] if not app: diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py index 9e5c35049d..677ed1bee6 100644 --- a/src/deployment/deploylib/registration.py +++ b/src/deployment/deploylib/registration.py @@ -160,6 +160,7 @@ class OnefuzzAppRole(Enum): ManagedNode = "ManagedNode" CliClient = "CliClient" UserAssignment = "UserAssignment" + UnmanagedNode = "UnmanagedNode" def register_application( @@ -309,21 +310,25 @@ def try_sp_create() -> None: registered_app_id = registered_app["appId"] app_id = app["appId"] + + + return registered_app + +def authorize_and_assign_role(onfuzz_app_id: UUID, registered_app_id: UUID, role: OnefuzzAppRole) -> None: def try_authorize_application(data: Any) -> None: authorize_application( UUID(registered_app_id), - UUID(app_id), + UUID(onfuzz_app_id), subscription_id=subscription_id, ) retry(try_authorize_application, "authorize application") def try_assign_instance_role(data: Any) -> None: - assign_instance_app_role(onefuzz_instance_name, name, subscription_id, approle) + assign_instance_app_role(onefuzz_instance_name, name, subscription_id, role) retry(try_assign_instance_role, "assingn role") - return registered_app def add_application_password( @@ -575,7 +580,7 @@ def assign_app_role( def assign_instance_app_role( - onefuzz_instance_name: str, + onefuzz_instance_app_id: UUID, application_name: str, subscription_id: str, app_role: OnefuzzAppRole, @@ -799,6 +804,15 @@ def main() -> None: cli_registration_parser.add_argument( "--registration_name", help="the name of the cli registration" ) + register_app_parser = subparsers.add_parser( + "register_app", parents=[parent_parser] + ) + register_app_parser.add_argument( + "--app_id", help="the application id to register" + ) + register_app_parser.add_argument( + "--role", help="the role of the application to register" + ) args = parser.parse_args() if args.verbose: @@ -821,6 +835,15 @@ def main() -> None: args.subscription_id, display_secret=True, ) + elif args.command == "register_app": + registration_name = args.registration_name or ("%s_unmanaged" % onefuzz_instance_name) + create_and_display_registration( + onefuzz_instance_name, + registration_name, + OnefuzzAppRole.UnmanagedNode, + args.subscription_id, + display_secret=True, + ) elif args.command == "assign_scaleset_role": assign_instance_app_role( onefuzz_instance_name, From 545a5ea34f9ba56dd7dd5afe46d31f85908f8c50 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Fri, 2 Sep 2022 13:04:18 -0700 Subject: [PATCH 02/52] fix tests --- .../ApiService/Functions/AgentRegistration.cs | 18 ++++++++++++++++++ .../ApiService/OneFuzzTypes/Requests.cs | 2 +- .../FunctionalTests/1f-api/Helpers.cs | 7 +------ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index 40255c2390..0168fe1ddf 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -90,6 +90,24 @@ private async Async.Task Post(HttpRequestData req) { var version = request.OkV.Version; var os = request.OkV.Os; + if (machineId == Guid.Empty) { + return await _context.RequestHandling.NotOk( + req, + new Error( + ErrorCode.INVALID_REQUEST, + new string[] { "'machine_id' query parameter must be provided" }), + "agent registration"); + } + + if (poolName is null) { + return await _context.RequestHandling.NotOk( + req, + new Error( + ErrorCode.INVALID_REQUEST, + new string[] { "'pool_name' query parameter must be provided" }), + "agent registration"); + } + _log.Info($"registration request: machine_id: {machineId} pool_name: {poolName} scaleset_id: {scalesetId} version: {version}"); var poolResult = await _context.PoolOperations.GetByName(poolName); if (!poolResult.IsOk) { diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 6c42b98e3b..75218b4851 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -294,7 +294,7 @@ Guid MachineId public record AgentRegistrationPost( PoolName PoolName, - Guid? ScalesetId, + Guid ScalesetId, Guid MachineId, string Version, Os? Os diff --git a/src/ApiService/FunctionalTests/1f-api/Helpers.cs b/src/ApiService/FunctionalTests/1f-api/Helpers.cs index 42e21a5e71..a899c13cd0 100644 --- a/src/ApiService/FunctionalTests/1f-api/Helpers.cs +++ b/src/ApiService/FunctionalTests/1f-api/Helpers.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Xunit; +using Xunit; namespace FunctionalTests { public class Helpers { From ced981df9f18ab7c0a5ffdf8ddd1290023d0df2c Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 26 Sep 2022 09:51:53 -0700 Subject: [PATCH 03/52] cleanup --- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 4 ++++ src/ApiService/IntegrationTests/JobsTests.cs | 2 +- src/ApiService/IntegrationTests/NodeTests.cs | 10 +++++----- src/ApiService/IntegrationTests/PoolTests.cs | 10 +++++----- src/ApiService/IntegrationTests/ScalesetTests.cs | 2 +- src/ApiService/IntegrationTests/TasksTests.cs | 2 +- src/ApiService/Tests/RemoveUserInfoTest.cs | 3 +-- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index a74b18aa17..f2f2e4fb15 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -167,6 +167,10 @@ public sealed override string ToString() { }; public record UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn, List Roles) { + + public UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn) : this(ApplicationId, ObjectId, Upn, new List()) { + } + public static UserInfo Create() { return new UserInfo(null, null, null, new List()); } diff --git a/src/ApiService/IntegrationTests/JobsTests.cs b/src/ApiService/IntegrationTests/JobsTests.cs index 6dc07d367c..33ec054a13 100644 --- a/src/ApiService/IntegrationTests/JobsTests.cs +++ b/src/ApiService/IntegrationTests/JobsTests.cs @@ -157,7 +157,7 @@ public async Async.Task Post_CreatesJob_AndContainer() { var func = new Jobs(auth, Context, Logger); // need user credentials to put into the job object - var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "upn", new List()); + var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var result = await func.Run(TestHttpRequestData.FromJson("POST", _config)); diff --git a/src/ApiService/IntegrationTests/NodeTests.cs b/src/ApiService/IntegrationTests/NodeTests.cs index d3cb13f709..4265113e83 100644 --- a/src/ApiService/IntegrationTests/NodeTests.cs +++ b/src/ApiService/IntegrationTests/NodeTests.cs @@ -163,7 +163,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -191,7 +191,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -221,7 +221,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -252,7 +252,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new NodeGet(MachineId: _machineId); @@ -281,7 +281,7 @@ await Context.InsertAll( var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); // override the found user credentials - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); // all of these operations use NodeGet diff --git a/src/ApiService/IntegrationTests/PoolTests.cs b/src/ApiService/IntegrationTests/PoolTests.cs index 2bb3097d7d..4afdc894dd 100644 --- a/src/ApiService/IntegrationTests/PoolTests.cs +++ b/src/ApiService/IntegrationTests/PoolTests.cs @@ -145,7 +145,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -167,7 +167,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Halt, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -189,7 +189,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); @@ -210,7 +210,7 @@ await Context.InsertAll( new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }); // needed for admin check // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); // need to override instance id @@ -241,7 +241,7 @@ await Context.InsertAll( new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null)); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); diff --git a/src/ApiService/IntegrationTests/ScalesetTests.cs b/src/ApiService/IntegrationTests/ScalesetTests.cs index ece3eaafe9..97f94752f1 100644 --- a/src/ApiService/IntegrationTests/ScalesetTests.cs +++ b/src/ApiService/IntegrationTests/ScalesetTests.cs @@ -71,7 +71,7 @@ public async Async.Task Search_AllScalesets_ReturnsEmptyIfNoneFound() { public async Async.Task Create_Scaleset() { var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); - // override the found user credentials + // override the found user credentials var userObjectId = Guid.NewGuid(); var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); diff --git a/src/ApiService/IntegrationTests/TasksTests.cs b/src/ApiService/IntegrationTests/TasksTests.cs index 4f43006185..d541781c46 100644 --- a/src/ApiService/IntegrationTests/TasksTests.cs +++ b/src/ApiService/IntegrationTests/TasksTests.cs @@ -55,7 +55,7 @@ public async Async.Task PoolIsRequired() { var func = new Tasks(Logger, auth, Context); // override the found user credentials - need these to check for admin - var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn", new List()); + var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: Guid.NewGuid(), "upn"); Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult.Ok(userInfo)); var req = new TaskCreate( diff --git a/src/ApiService/Tests/RemoveUserInfoTest.cs b/src/ApiService/Tests/RemoveUserInfoTest.cs index 4641e6aca1..84116c475e 100644 --- a/src/ApiService/Tests/RemoveUserInfoTest.cs +++ b/src/ApiService/Tests/RemoveUserInfoTest.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Text.Json; using Microsoft.OneFuzz.Service; using Xunit; @@ -9,7 +8,7 @@ public class RemoveUserInfoTest { [Fact] void TestSerilize() { - var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "test", new List()); + var userInfo = new UserInfo(Guid.NewGuid(), Guid.NewGuid(), "test"); var options = new JsonSerializerOptions(); options.Converters.Add(new RemoveUserInfo()); var serialized = JsonSerializer.Serialize(userInfo, options); From 6792e82a48cd8f9119f7c7d9314028205797e420 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 27 Sep 2022 08:54:35 -0700 Subject: [PATCH 04/52] more test fixes --- src/ApiService/ApiService/Functions/Jobs.cs | 2 +- .../ApiService/Functions/ReproVmss.cs | 2 +- src/ApiService/ApiService/Functions/Tasks.cs | 2 +- .../ApiService/OneFuzzTypes/Model.cs | 9 +------- .../ApiService/OneFuzzTypes/Requests.cs | 10 ++++---- src/ApiService/ApiService/UserCredentials.cs | 23 +++++++++++-------- .../onefuzzlib/EndpointAuthorization.cs | 4 ++-- .../Fakes/TestUserCredentials.cs | 9 ++++---- src/ApiService/Tests/RequestsTests.cs | 2 +- 9 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Jobs.cs b/src/ApiService/ApiService/Functions/Jobs.cs index bbdafe6d9a..97981c318e 100644 --- a/src/ApiService/ApiService/Functions/Jobs.cs +++ b/src/ApiService/ApiService/Functions/Jobs.cs @@ -47,7 +47,7 @@ private async Task Post(HttpRequestData req) { JobId: Guid.NewGuid(), State: JobState.Init, Config: cfg) { - UserInfo = userInfo.OkV, + UserInfo = userInfo.OkV.UserInfo, }; // create the job logs container diff --git a/src/ApiService/ApiService/Functions/ReproVmss.cs b/src/ApiService/ApiService/Functions/ReproVmss.cs index 80f3a30c4a..55ffd9613d 100644 --- a/src/ApiService/ApiService/Functions/ReproVmss.cs +++ b/src/ApiService/ApiService/Functions/ReproVmss.cs @@ -73,7 +73,7 @@ private async Async.Task Post(HttpRequestData req) { Path: create.Path, Duration: create.Duration); - var vm = await _context.ReproOperations.Create(cfg, userInfo.OkV); + var vm = await _context.ReproOperations.Create(cfg, userInfo.OkV.UserInfo); if (!vm.IsOk) { return await _context.RequestHandling.NotOk( req, diff --git a/src/ApiService/ApiService/Functions/Tasks.cs b/src/ApiService/ApiService/Functions/Tasks.cs index 7e4f58f8ec..ea7a461abe 100644 --- a/src/ApiService/ApiService/Functions/Tasks.cs +++ b/src/ApiService/ApiService/Functions/Tasks.cs @@ -139,7 +139,7 @@ private async Async.Task Post(HttpRequestData req) { } } - var task = await _context.TaskOperations.Create(cfg, cfg.JobId, userInfo.OkV); + var task = await _context.TaskOperations.Create(cfg, cfg.JobId, userInfo.OkV.UserInfo); if (!task.IsOk) { return await _context.RequestHandling.NotOk( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index f2f2e4fb15..86f4ef32f8 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -166,14 +166,7 @@ public sealed override string ToString() { } }; -public record UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn, List Roles) { - - public UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn) : this(ApplicationId, ObjectId, Upn, new List()) { - } - - public static UserInfo Create() { - return new UserInfo(null, null, null, new List()); - } +public record UserInfo(Guid? ApplicationId, Guid? ObjectId, String? Upn) { } public record TaskDetails( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 1ac54728c3..80ab0708ee 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -293,15 +293,15 @@ public record InstanceConfigUpdate( public record AgentRegistrationGet( - Guid MachineId + [property: Required] Guid MachineId ) : BaseRequest; public record AgentRegistrationPost( - PoolName PoolName, - Guid ScalesetId, - Guid MachineId, - string Version, + [property: Required] PoolName PoolName, + [property: Required] Guid ScalesetId, + [property: Required] Guid MachineId, + [property: Required] string Version, Os? Os ) : BaseRequest; diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index 827f808bd7..32f7420f9a 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -10,9 +10,12 @@ namespace Microsoft.OneFuzz.Service; public interface IUserCredentials { public string? GetBearerToken(HttpRequestData req); public string? GetAuthToken(HttpRequestData req); - public Task> ParseJwtToken(HttpRequestData req); + public Task> ParseJwtToken(HttpRequestData req); } +public record UserAuthInfo(UserInfo UserInfo, List Roles); + + public class UserCredentials : IUserCredentials { ILogTracer _log; IConfigOperations _instanceConfig; @@ -59,26 +62,26 @@ from t in r.AllowedAadTenants return OneFuzzResult.Ok(allowedAddTenantsQuery.ToArray()); } - public virtual async Task> ParseJwtToken(HttpRequestData req) { + public virtual async Task> ParseJwtToken(HttpRequestData req) { var authToken = GetAuthToken(req); if (authToken is null) { - return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find authorization token" }); + return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find authorization token" }); } else { var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken(authToken); var allowedTenants = await GetAllowedTenants(); if (allowedTenants.IsOk) { if (allowedTenants.OkV is not null && allowedTenants.OkV.Contains(token.Issuer)) { var userInfo = - token.Payload.Claims.Aggregate(UserInfo.Create(), (acc, claim) => { + token.Payload.Claims.Aggregate(new UserAuthInfo(new UserInfo(null, null, null), new List()), (acc, claim) => { switch (claim.Type) { case "oid": - return acc with { ObjectId = Guid.Parse(claim.Value) }; + return acc with { UserInfo = acc.UserInfo with { ObjectId = Guid.Parse(claim.Value) } }; case "appId": - return acc with { ApplicationId = Guid.Parse(claim.Value) }; + return acc with { UserInfo = acc.UserInfo with { ApplicationId = Guid.Parse(claim.Value) } }; case "upn": - return acc with { Upn = claim.Value }; + return acc with { UserInfo = acc.UserInfo with { Upn = claim.Value } }; case "roles": acc.Roles.Add(claim.Value); return acc; @@ -87,15 +90,15 @@ public virtual async Task> ParseJwtToken(HttpRequestData } }); - return OneFuzzResult.Ok(userInfo); + return OneFuzzResult.Ok(userInfo); } else { var tenantsStr = allowedTenants.OkV is null ? "null" : String.Join(';', allowedTenants.OkV!); _log.Error($"issuer not from allowed tenant. issuer: {token.Issuer} - tenants: {tenantsStr}"); - return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unauthorized AAD issuer" }); + return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unauthorized AAD issuer" }); } } else { _log.Error("Failed to get allowed tenants"); - return OneFuzzResult.Error(allowedTenants.ErrorV); + return OneFuzzResult.Error(allowedTenants.ErrorV); } } } diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index 200753ee41..0a6d63e813 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -44,7 +44,7 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu return await _context.RequestHandling.NotOk(req, tokenResult.ErrorV, "token verification", HttpStatusCode.Unauthorized); } - var token = tokenResult.OkV; + var token = tokenResult.OkV.UserInfo; if (await IsUser(token)) { if (!allowUser) { return await Reject(req, token); @@ -96,7 +96,7 @@ public async Async.Task CheckRequireAdmins(HttpRequestData re Errors: new string[] { "no instance configuration found " }); } - return CheckRequireAdminsImpl(config, tokenResult.OkV); + return CheckRequireAdminsImpl(config, tokenResult.OkV.UserInfo); } private static OneFuzzResultVoid CheckRequireAdminsImpl(InstanceConfig config, UserInfo userInfo) { diff --git a/src/ApiService/IntegrationTests/Fakes/TestUserCredentials.cs b/src/ApiService/IntegrationTests/Fakes/TestUserCredentials.cs index f04a3c9508..4ee641880f 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestUserCredentials.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestUserCredentials.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.OneFuzz.Service; @@ -8,12 +9,12 @@ namespace IntegrationTests.Fakes; sealed class TestUserCredentials : UserCredentials { - private readonly OneFuzzResult _tokenResult; + private readonly OneFuzzResult _tokenResult; public TestUserCredentials(ILogTracer log, IConfigOperations instanceConfig, OneFuzzResult tokenResult) : base(log, instanceConfig) { - _tokenResult = tokenResult; + _tokenResult = tokenResult.IsOk ? OneFuzzResult.Ok(new UserAuthInfo(tokenResult.OkV, new List())) : OneFuzzResult.Error(tokenResult.ErrorV); } - public override Task> ParseJwtToken(HttpRequestData req) => Async.Task.FromResult(_tokenResult); + public override Task> ParseJwtToken(HttpRequestData req) => Async.Task.FromResult(_tokenResult); } diff --git a/src/ApiService/Tests/RequestsTests.cs b/src/ApiService/Tests/RequestsTests.cs index dc7637e5c4..51cb0b13b0 100644 --- a/src/ApiService/Tests/RequestsTests.cs +++ b/src/ApiService/Tests/RequestsTests.cs @@ -76,7 +76,7 @@ public void EnsureRequiredAttributesExistsOnNonNullableRequestProperties(Type re // find appropriate parameter var param = requestType.GetConstructors().Single().GetParameters().Single(p => p.Name == property.Name); Assert.True(param.HasDefaultValue, - "For request types, all non-nullable properties should either have a default value, or the [Required] attribute." + $"type '{requestType}' is invalid. For request types, all non-nullable properties should either have a default value, or the [Required] attribute." ); } else { // it is required, okay From 8701c0f070268f8df284454683c90eb02b762c6e Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 27 Sep 2022 09:24:43 -0700 Subject: [PATCH 05/52] added comment regarding the assumption the agent makes to determine if the node is unmanaged --- src/agent/onefuzz-agent/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 56ff42c7b1..9d2dc3b633 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -278,6 +278,8 @@ async fn run_agent(config: StaticConfig) -> Result<()> { Ok(registration) => registration, Err(_) => { if scaleset.is_some() { + // todo: We are assuming here tht if we run in scaleset we are autamtivally managed. + // but an unmanaged node can run an external scaleset config::Registration::create_managed(config.clone()).await? } else { config::Registration::create_unmanaged(config.clone()).await? From 32b0fe79a529e1bb27c9f360a140cdc3cceb0c82 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 28 Sep 2022 13:10:15 -0700 Subject: [PATCH 06/52] build fix --- src/deployment/deploylib/registration.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py index 677ed1bee6..ca00e83f6a 100644 --- a/src/deployment/deploylib/registration.py +++ b/src/deployment/deploylib/registration.py @@ -310,11 +310,12 @@ def try_sp_create() -> None: registered_app_id = registered_app["appId"] app_id = app["appId"] - - return registered_app -def authorize_and_assign_role(onfuzz_app_id: UUID, registered_app_id: UUID, role: OnefuzzAppRole) -> None: + +def authorize_and_assign_role( + onfuzz_app_id: UUID, registered_app_id: UUID, role: OnefuzzAppRole +) -> None: def try_authorize_application(data: Any) -> None: authorize_application( UUID(registered_app_id), @@ -330,7 +331,6 @@ def try_assign_instance_role(data: Any) -> None: retry(try_assign_instance_role, "assingn role") - def add_application_password( password_name: str, app_object_id: UUID, subscription_id: str ) -> Tuple[str, str]: @@ -580,7 +580,7 @@ def assign_app_role( def assign_instance_app_role( - onefuzz_instance_app_id: UUID, + onefuzz_instance_name: str, application_name: str, subscription_id: str, app_role: OnefuzzAppRole, @@ -804,12 +804,8 @@ def main() -> None: cli_registration_parser.add_argument( "--registration_name", help="the name of the cli registration" ) - register_app_parser = subparsers.add_parser( - "register_app", parents=[parent_parser] - ) - register_app_parser.add_argument( - "--app_id", help="the application id to register" - ) + register_app_parser = subparsers.add_parser("register_app", parents=[parent_parser]) + register_app_parser.add_argument("--app_id", help="the application id to register") register_app_parser.add_argument( "--role", help="the role of the application to register" ) @@ -836,7 +832,9 @@ def main() -> None: display_secret=True, ) elif args.command == "register_app": - registration_name = args.registration_name or ("%s_unmanaged" % onefuzz_instance_name) + registration_name = args.registration_name or ( + "%s_unmanaged" % onefuzz_instance_name + ) create_and_display_registration( onefuzz_instance_name, registration_name, From 1788e55d6afae29500bbb34f0266da9fc84dffb9 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Fri, 7 Oct 2022 13:05:42 -0700 Subject: [PATCH 07/52] docker file --- .../deploy-agent-in-docker/linux/Dockerfile | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 contrib/deploy-agent-in-docker/linux/Dockerfile diff --git a/contrib/deploy-agent-in-docker/linux/Dockerfile b/contrib/deploy-agent-in-docker/linux/Dockerfile new file mode 100644 index 0000000000..c012108802 --- /dev/null +++ b/contrib/deploy-agent-in-docker/linux/Dockerfile @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04 + +# Pull Request that contains OneFuzz release-artifacts +# used to create the Docker container +ARG RELEASE_VERSION + +RUN apt-get update +RUN apt-get install --yes --quiet curl \ + unzip \ + wget + +# creating a dummy sudo command +RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo + +RUN wget https://github.com/microsoft/onefuzz/releases/download/${RELEASE_VERSION}/onefuzz-deployment-${RELEASE_VERSION}.zip && \ + unzip -j onefuzz-deployment-${RELEASE_VERSION}.zip tools/linux/* -d onefuzz-install/ && \ + cd onefuzz-install && \ + chmod +x ./onefuzz-agent && \ + chmod +x ./run.sh && \ + chmod +x ./setup.sh + +RUN cd onefuzz-install && ./setup.sh + +ENTRYPOINT onefuzz/run.sh \ No newline at end of file From 5fc5b8a1997aaa1b6b386d3bd06d948242baa5c4 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 18 Oct 2022 10:17:58 -0700 Subject: [PATCH 08/52] build fix --- src/ApiService/ApiService/UserCredentials.cs | 4 +-- src/deployment/deploylib/registration.py | 32 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index 926253c93a..01a979ce3a 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -94,11 +94,11 @@ public virtual async Task> ParseJwtToken(HttpRequest } else { var tenantsStr = allowedTenants.OkV is null ? "null" : String.Join(';', allowedTenants.OkV!); _log.Error($"issuer not from allowed tenant. issuer: {token.Issuer:Tag:Issuer} - tenants: {tenantsStr:Tag:Tenants}"); - return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unauthorized AAD issuer" }); + return OneFuzzResult.Error(ErrorCode.INVALID_REQUEST, new[] { "unauthorized AAD issuer" }); } } else { _log.Error($"Failed to get allowed tenants due to {allowedTenants.ErrorV:Tag:Error}"); - return OneFuzzResult.Error(allowedTenants.ErrorV); + return OneFuzzResult.Error(allowedTenants.ErrorV); } } } diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py index ca00e83f6a..5763b49552 100644 --- a/src/deployment/deploylib/registration.py +++ b/src/deployment/deploylib/registration.py @@ -313,22 +313,22 @@ def try_sp_create() -> None: return registered_app -def authorize_and_assign_role( - onfuzz_app_id: UUID, registered_app_id: UUID, role: OnefuzzAppRole -) -> None: - def try_authorize_application(data: Any) -> None: - authorize_application( - UUID(registered_app_id), - UUID(onfuzz_app_id), - subscription_id=subscription_id, - ) - - retry(try_authorize_application, "authorize application") - - def try_assign_instance_role(data: Any) -> None: - assign_instance_app_role(onefuzz_instance_name, name, subscription_id, role) - - retry(try_assign_instance_role, "assingn role") +# def authorize_and_assign_role( +# onfuzz_app_id: UUID, registered_app_id: UUID, role: OnefuzzAppRole, subscription_id:Optional[str] +# ) -> None: +# def try_authorize_application(data: Any) -> None: +# authorize_application( +# registered_app_id, +# onfuzz_app_id, +# subscription_id=subscription_id, +# ) + +# retry(try_authorize_application, "authorize application") + +# def try_assign_instance_role(data: Any) -> None: +# assign_instance_app_role(onefuzz_instance_name, name, subscription_id, role) + +# retry(try_assign_instance_role, "assingn role") def add_application_password( From 245381f635802cca27856e242a4a4450c8af4078 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 18 Oct 2022 11:43:12 -0700 Subject: [PATCH 09/52] fix unit tests --- src/ApiService/ApiService/Functions/AgentRegistration.cs | 9 +++++++++ src/ApiService/ApiService/OneFuzzTypes/Requests.cs | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index d7e370f0b3..72b4788682 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -37,6 +37,15 @@ private async Async.Task Get(HttpRequestData req) { var machineId = request.OkV.MachineId; + if (machineId == Guid.Empty) { + return await _context.RequestHandling.NotOk( + req, + new Error( + ErrorCode.INVALID_REQUEST, + new string[] { "'machine_id' query parameter must be provided" }), + "agent registration"); + } + var agentNode = await _context.NodeOperations.GetByMachineId(machineId); if (agentNode is null) { return await _context.RequestHandling.NotOk( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 80ab0708ee..aeae7f0dde 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -301,7 +301,7 @@ public record AgentRegistrationPost( [property: Required] PoolName PoolName, [property: Required] Guid ScalesetId, [property: Required] Guid MachineId, - [property: Required] string Version, - Os? Os + Os? Os, + [property: Required] string Version = "1.0.0" ) : BaseRequest; From 386d4ceccf60798c5e8c793faa2fdf9bf310fdcc Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 18 Oct 2022 16:11:21 -0700 Subject: [PATCH 10/52] fix getting unmnaged pool config --- .../ApiService/Functions/GetPoolConfig.cs | 58 ------------------- src/cli/onefuzz/api.py | 21 +------ 2 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 src/ApiService/ApiService/Functions/GetPoolConfig.cs diff --git a/src/ApiService/ApiService/Functions/GetPoolConfig.cs b/src/ApiService/ApiService/Functions/GetPoolConfig.cs deleted file mode 100644 index 266a7dab6b..0000000000 --- a/src/ApiService/ApiService/Functions/GetPoolConfig.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Net; -using System.Security.Claims; -using System.Threading.Tasks; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Azure.Functions.Worker.Http; - -namespace Microsoft.OneFuzz.Service.Functions; - -public class GetPoolConfig { - private readonly ILogTracer _log; - private readonly IEndpointAuthorization _auth; - private readonly IOnefuzzContext _context; - - public GetPoolConfig(ILogTracer log, IEndpointAuthorization auth, IOnefuzzContext context) { - _log = log; - _auth = auth; - _context = context; - } - - [Function("GetPoolConfig")] - public Async.Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", Route = "pool/getconfig")] HttpRequestData req, ClaimsPrincipal principal) - => _auth.CallIfUser(req, r => r.Method switch { - "GET" => Get(r), - var m => throw new InvalidOperationException("Unsupported HTTP method {m}"), - }); - - private async Task Get(HttpRequestData req) { - var request = await RequestHandling.ParseRequest(req); - if (!request.IsOk) { - return await _context.RequestHandling.NotOk(req, request.ErrorV, "pool get config"); - } - - var search = request.OkV; - OneFuzzResult poolResult; - if (search.PoolId is Guid poolId) { - poolResult = await _context.PoolOperations.GetById(poolId); - } else if (search.Name is PoolName name) { - poolResult = await _context.PoolOperations.GetByName(name); - } else { - return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "missing pool name or id" }), "pool get config"); - } - - if (!poolResult.IsOk) { - return await _context.RequestHandling.NotOk(req, poolResult.ErrorV, context: search.ToString()); - } - - if (!poolResult.OkV.Managed) { - return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "Pool is a managed pool" }), context: search.ToString()); - } - var poolConfig = await _context.Extensions.CreatePoolConfig(poolResult.OkV); - - var response = req.CreateResponse(HttpStatusCode.OK); - await response.WriteAsJsonAsync(poolConfig); - return response; - } - - -} diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 2b3080dc04..67aa7c9a66 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1213,13 +1213,7 @@ def get_config(self, pool_name: primitives.PoolName) -> models.AgentConfig: if pool.config is None: raise Exception("Missing AgentConfig in response") - config = pool.config - config.client_credentials = models.ClientCredentials( # nosec - bandit consider this a hard coded password - client_id=pool.client_id, - client_secret="", - ) - - return config + return pool.config def shutdown(self, name: str, *, now: bool = False) -> responses.BoolResult: expanded_name = self._disambiguate( @@ -1251,19 +1245,6 @@ def list( "GET", models.Pool, data=requests.PoolSearch(state=state) ) - def get_unamaged_config(self, name: str) -> models.AgentConfig: - self.logger.debug("get details on a specific pool") - expanded_name = self._disambiguate( - "pool name", name, lambda x: False, lambda: [x.name for x in self.list()] - ) - - return self._req_model( - "GET", - models.AgentConfig, - data=requests.PoolSearch(name=expanded_name), - alternate_endpoint="pool/getconfig", - ) - class Node(Endpoint): """Interact with nodes""" From a57fe0464c91557dddc9816e3ee610b72ea6d78a Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 10 Nov 2022 10:43:56 -0800 Subject: [PATCH 11/52] add is_unmanaged field --- src/agent/onefuzz-agent/src/config.rs | 11 +++++++++-- src/agent/onefuzz-agent/src/main.rs | 8 +++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index d3bb0e8d88..3eb56c94ac 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -34,6 +34,9 @@ pub struct StaticConfig { pub heartbeat_queue: Option, pub instance_id: Uuid, + + #[serde(default)] + pub is_unmanaged: bool, } // Temporary shim type to bridge the current service-provided config. @@ -54,6 +57,9 @@ struct RawStaticConfig { pub heartbeat_queue: Option, pub instance_id: Uuid, + + #[serde(default)] + pub is_unmanaged: bool, } impl StaticConfig { @@ -83,6 +89,7 @@ impl StaticConfig { instance_telemetry_key: config.instance_telemetry_key, heartbeat_queue: config.heartbeat_queue, instance_id: config.instance_id, + is_unmanaged: config.is_unmanaged, }; Ok(config) @@ -103,6 +110,7 @@ impl StaticConfig { let multi_tenant_domain = std::env::var("ONEFUZZ_MULTI_TENANT_DOMAIN").ok(); let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; + let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { Some(Url::parse(&key)?) @@ -142,6 +150,7 @@ impl StaticConfig { microsoft_telemetry_key, heartbeat_queue, instance_id, + is_unmanaged }) } @@ -216,8 +225,6 @@ impl Registration { .append_pair("version", env!("ONEFUZZ_VERSION")) .append_pair("os", std::env::consts::OS); - //todo: add os version? - if managed { let scaleset = onefuzz::machine_id::get_scaleset_name().await?; match scaleset { diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 9d2dc3b633..14a8852e78 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -277,12 +277,10 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let registration = match config::Registration::load_existing(config.clone()).await { Ok(registration) => registration, Err(_) => { - if scaleset.is_some() { - // todo: We are assuming here tht if we run in scaleset we are autamtivally managed. - // but an unmanaged node can run an external scaleset - config::Registration::create_managed(config.clone()).await? - } else { + if config.is_unmanaged { config::Registration::create_unmanaged(config.clone()).await? + } else { + config::Registration::create_managed(config.clone()).await? } } }; From 5daf10d8443d53ddc9bc545f3dd62905a73a92e1 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 10 Nov 2022 10:51:18 -0800 Subject: [PATCH 12/52] setting the IsUnmanagd field in the config --- src/ApiService/ApiService/Functions/Pool.cs | 3 ++- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 3 ++- src/ApiService/ApiService/onefuzzlib/Extension.cs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Pool.cs b/src/ApiService/ApiService/Functions/Pool.cs index d9fefc49fe..40160c02b8 100644 --- a/src/ApiService/ApiService/Functions/Pool.cs +++ b/src/ApiService/ApiService/Functions/Pool.cs @@ -133,7 +133,8 @@ private async Task Populate(PoolGetResult p, bool skipSummaries = HeartbeatQueue: queueSas, InstanceId: instanceId, ClientCredentials: null, - MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain) + MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, + IsUnamanaged: !p.Managed) }; } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index f72cd95ec8..e1bb9e583e 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -679,7 +679,8 @@ public record AgentConfig( string? InstanceTelemetryKey, string? MicrosoftTelemetryKey, string? MultiTenantDomain, - Guid InstanceId + Guid InstanceId, + bool? IsUnamanaged ); diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index 973b6d2a19..e1063b7118 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -232,7 +232,8 @@ public async Task CreatePoolConfig(Pool pool) { InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey, MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, - InstanceId: instanceId + InstanceId: instanceId, + IsUnamanaged: !pool.Managed ); return config; } From 26132a71c5c28791ee50654723f966a480e19b39 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 10 Nov 2022 11:51:20 -0800 Subject: [PATCH 13/52] format --- src/agent/onefuzz-agent/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 3eb56c94ac..e67ce4d775 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -110,7 +110,7 @@ impl StaticConfig { let multi_tenant_domain = std::env::var("ONEFUZZ_MULTI_TENANT_DOMAIN").ok(); let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; - let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); + let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { Some(Url::parse(&key)?) @@ -150,7 +150,7 @@ impl StaticConfig { microsoft_telemetry_key, heartbeat_queue, instance_id, - is_unmanaged + is_unmanaged, }) } From 98fe58b0e11b1ae789c108d81ac2d68a294766fe Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 10 Nov 2022 13:53:32 -0800 Subject: [PATCH 14/52] format --- src/deployment/deploylib/registration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py index 2f7881d5f6..2bd6a70e95 100644 --- a/src/deployment/deploylib/registration.py +++ b/src/deployment/deploylib/registration.py @@ -323,7 +323,6 @@ def try_sp_create() -> None: registered_app_id = registered_app["appId"] app_id = app["appId"] - authorize_and_assign_role(app_id, registered_app_id, approle, subscription_id) return registered_app From c8c0eb366211ff8cc9b647901aef4ccdde7911ea Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Sun, 13 Nov 2022 22:53:08 -0800 Subject: [PATCH 15/52] unmanaged node deployment script --- src/deployment/unmanaged/__init__.py | 0 .../unmanaged/deploy_unamaged_scaleset.py | 214 ++++++++ src/deployment/unmanaged/requirements.txt | 5 + src/deployment/unmanaged/resource_group.bicep | 0 .../unmanaged/scaleset_template_windows.bicep | 496 ++++++++++++++++++ 5 files changed, 715 insertions(+) create mode 100644 src/deployment/unmanaged/__init__.py create mode 100644 src/deployment/unmanaged/deploy_unamaged_scaleset.py create mode 100644 src/deployment/unmanaged/requirements.txt create mode 100644 src/deployment/unmanaged/resource_group.bicep create mode 100644 src/deployment/unmanaged/scaleset_template_windows.bicep diff --git a/src/deployment/unmanaged/__init__.py b/src/deployment/unmanaged/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/deployment/unmanaged/deploy_unamaged_scaleset.py b/src/deployment/unmanaged/deploy_unamaged_scaleset.py new file mode 100644 index 0000000000..3d89eef5cf --- /dev/null +++ b/src/deployment/unmanaged/deploy_unamaged_scaleset.py @@ -0,0 +1,214 @@ +# This script is a helper to deploy a scale set of unmanaged machines + +# deploy the unamanged template +# optionally upload the deployment files +# the script should reference the deployment files + +import argparse +import json +import logging +import os +import subprocess +import sys +import tempfile +import uuid +import zipfile +from typing import Any, List, Optional, cast +from zipfile import ZipFile + +from azure.cli.core import get_default_cli +from azure.common.credentials import get_cli_profile +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError, ResourceExistsError +from azure.identity import AzureCliCredential +from azure.mgmt.resource import ResourceManagementClient, SubscriptionClient +from azure.mgmt.resource.resources.models import (Deployment, DeploymentMode, + DeploymentProperties) +from azure.mgmt.storage import StorageManagementClient +from azure.storage.blob import (BlobServiceClient, ContainerSasPermissions, + generate_container_sas) +from onefuzz.api import Onefuzz + +from azure.mgmt.storage.models import ( + AccessTier, + Kind, + Sku, + SkuName, + StorageAccountCreateParameters, +) + +logger = logging.getLogger("deploy") +def bicep_to_arm(bicep_template: str, output_path: str = "azuredeploy-bicep.json") -> str: + return subprocess.check_output( # nosec + [ "az", + "bicep", + "build", + "--file", + bicep_template, + "--stdout" + ], shell=True + ) + + +class Deployer: + arm_template = "scaleset_template_windows.bicep" + def __init__(self, + resource_group: str, + location: str, + subscription_id: Optional[str], + arm_template: str = "scaleset_template_windows.bicep" , + scaleset_size: int = 1, + ): + self.arm_template = arm_template + self.resource_group = resource_group + self.location = location + self.scaleset_size = scaleset_size + self.storage_account = f"{self.resource_group}sa".replace("-", "").replace("_", "") + + if subscription_id: + self.subscription_id = subscription_id + else: + profile = get_cli_profile() + self.subscription_id = cast(str, profile.get_subscription_id()) + + + pass + + def get_template(path: str) -> Any: + output = bicep_to_arm(path) + return json.loads(output) + + def deploy(self): + logger.info("deploying") + template = Deployer.get_template(self.arm_template) + logger.info("deploying 2") + credential = AzureCliCredential() + client = ResourceManagementClient( + credential, subscription_id=self.subscription_id + ) + logger.info("deploying 3") + + # client.resource_groups.get(self.resource_group) + + storageClient = StorageManagementClient(credential, subscription_id=self.subscription_id) + try: + logger.info("checking for storage account") + prop = storageClient.storage_accounts.get_properties(self.resource_group, self.storage_account) + logger.info("storage account exists") + except HttpResponseError as e: + logger.info("storage account does not exist creating a new one") + params = StorageAccountCreateParameters( + sku=Sku(name=SkuName.PREMIUM_LRS), + kind=Kind.block_blob_storage, + location=self.location, + access_tier=AccessTier.hot, + allow_blob_public_access=False, + minimum_tls_version="TLS1_2", + ) + r = storageClient.storage_accounts.begin_create(self.resource_group, self.storage_account, params).result() + logger.info("storage account created") + + + file_uris = self.upload_tools(storageClient) + + logger.info("deploying scaleset") + client.resource_groups.create_or_update( + self.resource_group, {"location": self.location} + ) + + params = { + "scaleset_name" : { "value": self.resource_group }, + "location" : { "value": self.location }, + "networkSecurityGroups_name" : { "value": "nsg" }, + "adminUsername" : { "value": "onefuzz" }, + "capacity" : { "value": self.scaleset_size }, + "adminPassword": { "value": str(uuid.uuid4()) }, + "file_uris": { "value": file_uris }, + + + } + + deployment = Deployment( + properties=DeploymentProperties( + mode=DeploymentMode.incremental, template=template, parameters=params + ) + ) + + result = client.deployments.begin_create_or_update( + self.resource_group, str(uuid.uuid4()), deployment + ).result() + + if result.properties.provisioning_state != "Succeeded": + logging.Logger.error( + "error deploying: %s", + json.dumps(result.as_dict(), indent=4, sort_keys=True), + ) + sys.exit(1) + + def upload_tools(self, storageClient: StorageManagementClient) -> List[str]: + logger.info("downloading tools") + onefuzz = Onefuzz() + uris = [] + with tempfile.TemporaryDirectory() as tmpDir: + zip_path = os.path.join(tmpDir, "tools.zip") + extracted_path = os.path.join(tmpDir, "tools") + onefuzz.tools.get(zip_path) + # extract zip + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(extracted_path) + + blob_client: BlobServiceClient = BlobServiceClient.from_connection_string(storageClient.storage_accounts.list_keys(self.resource_group, self.storage_account).keys[0].value) + try: + container_client = blob_client.create_container("tools") + except ResourceExistsError: + container_client = blob_client.get_container_client("tools") + pass + + logger.info("uploading files") + + for file in os.listdir(extracted_path): + logger.debug(f"uploading {file}") + blob_client = container_client.upload_blob(file, os.path.join(extracted_path, file)) + print (f"uploaded {blob_client.url}") + uris.append(blob_client.url) + + return uris + +def arg_file(arg: str) -> str: + if not os.path.isfile(arg): + raise argparse.ArgumentTypeError("not a file: %s" % arg) + return arg + +def main() -> None: + + formatter = argparse.ArgumentDefaultsHelpFormatter + parser = argparse.ArgumentParser(formatter_class=formatter) + parser.add_argument("location") + parser.add_argument("resource_group") + # parser.add_argument("owner") + # parser.add_argument("nsg_config") + parser.add_argument( + "--bicep-template", + type=arg_file, + default="scaleset_template_windows.bicep", + help="(default: %(default)s)", + ) + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument( + "--subscription_id", + type=str, + ) + args = parser.parse_args() + if args.verbose: + level = logging.DEBUG + else: + level = logging.WARN + + logging.basicConfig(level=level) + logging.getLogger("deploy").setLevel(logging.INFO) + + deployer = Deployer(args.resource_group, args.location, args.subscription_id, args.bicep_template) + deployer.deploy() + + +if __name__ == "__main__": + main() diff --git a/src/deployment/unmanaged/requirements.txt b/src/deployment/unmanaged/requirements.txt new file mode 100644 index 0000000000..fd910cbe4b --- /dev/null +++ b/src/deployment/unmanaged/requirements.txt @@ -0,0 +1,5 @@ +azure-identity==1.10.0 +azure-mgmt-resource==21.1.0b1 +azure-mgmt-storage==20.1.0 +azure-storage-blob==12.13.1 +onefuzz==0.0.0 diff --git a/src/deployment/unmanaged/resource_group.bicep b/src/deployment/unmanaged/resource_group.bicep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/deployment/unmanaged/scaleset_template_windows.bicep b/src/deployment/unmanaged/scaleset_template_windows.bicep new file mode 100644 index 0000000000..0a4fa54773 --- /dev/null +++ b/src/deployment/unmanaged/scaleset_template_windows.bicep @@ -0,0 +1,496 @@ +param scaleset_name string +param location string +param networkSecurityGroups_name string +param adminUsername string = 'onefuzz' +param capacity int = 1 +param vmSize string = 'Standard_D2s_v3' +param tier string = 'Standard' +param fileUris array = [] +@secure() +param adminPassword string + +resource scaleset 'Microsoft.Compute/virtualMachineScaleSets@2022-03-01' = { + name: scaleset_name + location: location + tags: { + azsecpack: 'nonprod' + 'platformsettings.host_environment.service.platform_optedin_for_rootcerts': 'true' + } + sku: { + name: vmSize + tier: tier + capacity: capacity + } + identity: { + type: 'SystemAssigned' + } + properties: { + singlePlacementGroup: false + orchestrationMode: 'Uniform' + upgradePolicy: { + mode: 'Manual' + } + virtualMachineProfile: { + osProfile: { + computerNamePrefix: 'node' + adminUsername: adminUsername + adminPassword: adminPassword + windowsConfiguration: { + provisionVMAgent: true + enableAutomaticUpdates: true + } + secrets: [] + allowExtensionOperations: true + } + storageProfile: { + osDisk: { + osType: 'Windows' + createOption: 'FromImage' + caching: 'None' + managedDisk: { + storageAccountType: 'Premium_LRS' + } + diskSizeGB: 127 + } + imageReference: { + publisher: 'MicrosoftWindowsDesktop' + offer: 'Windows-10' + sku: 'win10-21h2-pro' + version: 'latest' + } + } + networkProfile: { + networkInterfaceConfigurations: [ + { + name: 'onefuzz-nic' + properties: { + primary: true + enableAcceleratedNetworking: false + dnsSettings: { + dnsServers: [] + } + enableIPForwarding: false + ipConfigurations: [ + { + name: 'onefuzz-ip-config' + properties: { + subnet: { + id: subnet.id + } + privateIPAddressVersion: 'IPv4' + } + } + ] + } + } + ] + } + extensionProfile: { + extensions: [ + { + name: 'OMSExtension' + properties: { + autoUpgradeMinorVersion: true + enableAutomaticUpgrade: false + publisher: 'Microsoft.EnterpriseCloud.Monitoring' + type: 'MicrosoftMonitoringAgent' + typeHandlerVersion: '1.0' + settings: { + workspaceId: '77da01e8-838a-41d9-a6d0-c8b180f68904' + } + } + } + { + name: 'DependencyAgentWindows' + properties: { + autoUpgradeMinorVersion: true + publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' + type: 'DependencyAgentWindows' + typeHandlerVersion: '9.5' + settings: { + } + } + } + { + name: 'Microsoft.Azure.Geneva.GenevaMonitoring' + properties: { + autoUpgradeMinorVersion: true + enableAutomaticUpgrade: true + publisher: 'Microsoft.Azure.Geneva' + type: 'GenevaMonitoring' + typeHandlerVersion: '2.0' + settings: { + } + } + } + { + name: 'CustomScriptExtension' + properties: { + autoUpgradeMinorVersion: true + forceUpdateTag: 'e2dc833a-58d2-4f96-a462-4e079f5fc1e0' + publisher: 'Microsoft.Compute' + type: 'CustomScriptExtension' + typeHandlerVersion: '1.9' + settings: { + commandToExecute: 'powershell -ExecutionPolicy Unrestricted -File win64/setup.ps1 -mode fuzz' + fileUris: fileUris + } + } + } + { + name: 'Microsoft.Azure.Security.AntimalwareSignature.AntimalwareConfiguration' + properties: { + autoUpgradeMinorVersion: true + enableAutomaticUpgrade: true + publisher: 'Microsoft.Azure.Security.AntimalwareSignature' + type: 'AntimalwareConfiguration' + typeHandlerVersion: '2.0' + settings: { + } + } + } + ] + } + priority: 'Regular' + } + overprovision: false + doNotRunExtensionsOnOverprovisionedVMs: false + } +} + +// resource scaleset_CustomScriptExtension 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { +// parent: scaleset +// name: 'CustomScriptExtension' +// properties: { +// autoUpgradeMinorVersion: true +// forceUpdateTag: 'e2dc833a-58d2-4f96-a462-4e079f5fc1e0' +// publisher: 'Microsoft.Compute' +// type: 'CustomScriptExtension' +// typeHandlerVersion: '1.9' +// settings: { +// commandToExecute: 'powershell -ExecutionPolicy Unrestricted -File win64/setup.ps1 -mode fuzz' +// fileUris: [ +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/vm-scripts/pool_windows/config.json' +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/vm-scripts/be10f6a5-4efd-432f-be1a-25cc18e77900/scaleset-setup.ps1' +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/vm-scripts/managed.ps1' +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/tools/win64/azcopy.exe' +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/tools/win64/setup.ps1' +// // 'https://func7tkgjsiivmq6i.blob.core.windows.net/tools/win64/onefuzz.ps1' +// ] +// } +// } +// } + +// resource scaleset_DependencyAgentWindows 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { +// parent: scaleset +// name: 'DependencyAgentWindows' +// properties: { +// autoUpgradeMinorVersion: true +// publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' +// type: 'DependencyAgentWindows' +// typeHandlerVersion: '9.5' +// settings: { +// } +// } +// } + +// resource scaleset_Microsoft_Azure_Geneva_GenevaMonitoring 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { +// parent: scaleset +// name: 'Microsoft.Azure.Geneva.GenevaMonitoring' +// properties: { +// autoUpgradeMinorVersion: true +// enableAutomaticUpgrade: true +// publisher: 'Microsoft.Azure.Geneva' +// type: 'GenevaMonitoring' +// typeHandlerVersion: '2.0' +// settings: { +// } +// } +// } + +// resource scaleset_Microsoft_Azure_Security_AntimalwareSignature_AntimalwareConfiguration 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { +// parent: scaleset +// name: 'Microsoft.Azure.Security.AntimalwareSignature.AntimalwareConfiguration' +// properties: { +// autoUpgradeMinorVersion: true +// enableAutomaticUpgrade: true +// publisher: 'Microsoft.Azure.Security.AntimalwareSignature' +// type: 'AntimalwareConfiguration' +// typeHandlerVersion: '2.0' +// settings: { +// } +// } +// } + +// resource scaleset_OMSExtension 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { +// parent: scaleset +// name: 'OMSExtension' +// properties: { +// autoUpgradeMinorVersion: true +// enableAutomaticUpgrade: false +// publisher: 'Microsoft.EnterpriseCloud.Monitoring' +// type: 'MicrosoftMonitoringAgent' +// typeHandlerVersion: '1.0' +// settings: { +// workspaceId: '77da01e8-838a-41d9-a6d0-c8b180f68904' +// } +// } +// } + +resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2022-01-01' = { + name: networkSecurityGroups_name + location: location + tags: { + } + properties: { + securityRules: [ + { + name: 'NRMS-Rule-109' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'DO NOT DELETE - Will result in ICM Sev 2 - Azure Core Security, see aka.ms/cainsgpolicy' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 109 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '119' + '137' + '138' + '139' + '161' + '162' + '389' + '636' + '2049' + '2301' + '2381' + '3268' + '5800' + '5900' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + { + name: 'NRMS-Rule-104' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'Created by Azure Core Security managed policy, rule can be deleted but do not change source ips, please see aka.ms/cainsgpolicy' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'CorpNetSaw' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 104 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + { + name: 'NRMS-Rule-105' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'DO NOT DELETE - Will result in ICM Sev 2 - Azure Core Security, see aka.ms/cainsgpolicy' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 105 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '1433' + '1434' + '3306' + '4333' + '5432' + '6379' + '7000' + '7001' + '7199' + '9042' + '9160' + '9300' + '16379' + '26379' + '27017' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + { + name: 'NRMS-Rule-107' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'DO NOT DELETE - Will result in ICM Sev 2 - Azure Core Security, see aka.ms/cainsgpolicy' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 107 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '23' + '135' + '445' + '5985' + '5986' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + { + name: 'NRMS-Rule-106' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'DO NOT DELETE - Will result in ICM Sev 2 - Azure Core Security, see aka.ms/cainsgpolicy' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 106 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '22' + '3389' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + { + name: 'NRMS-Rule-108' + type: 'Microsoft.Network/networkSecurityGroups/securityRules' + properties: { + description: 'DO NOT DELETE - Will result in ICM Sev 2 - Azure Core Security, see aka.ms/cainsgpolicy' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 108 + direction: 'Inbound' + sourcePortRanges: [] + destinationPortRanges: [ + '13' + '17' + '19' + '53' + '69' + '111' + '123' + '512' + '514' + '593' + '873' + '1900' + '5353' + '11211' + ] + sourceAddressPrefixes: [] + destinationAddressPrefixes: [] + } + } + ] + } +} + +resource Vnet 'Microsoft.Network/virtualNetworks@2022-05-01' = { + name: 'vnet' + + location: location + properties: { + addressSpace: { + addressPrefixes: [ + '10.0.0.0/8' + ] + } + subnets: [ + { + name: 'subnet' + properties: { + addressPrefix: '10.0.0.0/16' + networkSecurityGroup: { + id: networkSecurityGroup.id + } + delegations: [] + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } + type: 'Microsoft.Network/virtualNetworks/subnets' + } + ] + virtualNetworkPeerings: [] + enableDdosProtection: false + } +} + +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2022-05-01' = { + name: 'vnet/subnet' + properties: { + addressPrefix: '10.0.0.0/16' + networkSecurityGroup: { + id: networkSecurityGroup.id + } + delegations: [] + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } + dependsOn: [ + Vnet + ] +} + +// resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { +// name: storage_acc +// location: location +// sku: { +// name: 'Standard_LRS' +// } +// kind: 'StorageV2' +// properties: { +// supportsHttpsTrafficOnly: true +// networkAcls: { +// bypass: 'AzureServices' +// defaultAction: 'Allow' +// ipRules: [] +// virtualNetworkRules: [] +// } +// encryption: { +// services: { +// blob: { +// enabled: true +// } +// file: { +// enabled: true +// } +// queue: { +// enabled: true +// } +// table: { +// enabled: true +// } +// } +// keySource: 'Microsoft.Storage' +// } +// accessTier: 'Hot' +// } +// } + From 40ac87f16d09457ffb02e2ab57d3450998404f36 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 14 Nov 2022 10:50:34 -0800 Subject: [PATCH 16/52] format --- src/deployment/unmanaged/__init__.py | 0 .../unmanaged/deploy_unamaged_scaleset.py | 114 ++++++++++-------- 2 files changed, 65 insertions(+), 49 deletions(-) delete mode 100644 src/deployment/unmanaged/__init__.py diff --git a/src/deployment/unmanaged/__init__.py b/src/deployment/unmanaged/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/deployment/unmanaged/deploy_unamaged_scaleset.py b/src/deployment/unmanaged/deploy_unamaged_scaleset.py index 3d89eef5cf..8c271aec17 100644 --- a/src/deployment/unmanaged/deploy_unamaged_scaleset.py +++ b/src/deployment/unmanaged/deploy_unamaged_scaleset.py @@ -14,20 +14,21 @@ import uuid import zipfile from typing import Any, List, Optional, cast -from zipfile import ZipFile -from azure.cli.core import get_default_cli from azure.common.credentials import get_cli_profile -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError, ResourceExistsError +from azure.core.exceptions import ( + HttpResponseError, + ResourceExistsError, + # ResourceNotFoundError, +) from azure.identity import AzureCliCredential -from azure.mgmt.resource import ResourceManagementClient, SubscriptionClient -from azure.mgmt.resource.resources.models import (Deployment, DeploymentMode, - DeploymentProperties) +from azure.mgmt.resource import ResourceManagementClient +from azure.mgmt.resource.resources.models import ( + Deployment, + DeploymentMode, + DeploymentProperties, +) from azure.mgmt.storage import StorageManagementClient -from azure.storage.blob import (BlobServiceClient, ContainerSasPermissions, - generate_container_sas) -from onefuzz.api import Onefuzz - from azure.mgmt.storage.models import ( AccessTier, Kind, @@ -35,34 +36,39 @@ SkuName, StorageAccountCreateParameters, ) +from azure.storage.blob import ( + BlobServiceClient, +) +from onefuzz.api import Onefuzz logger = logging.getLogger("deploy") -def bicep_to_arm(bicep_template: str, output_path: str = "azuredeploy-bicep.json") -> str: - return subprocess.check_output( # nosec - [ "az", - "bicep", - "build", - "--file", - bicep_template, - "--stdout" - ], shell=True - ) + + +def get_template(path: str) -> Any: + json_template = subprocess.check_output( # nosec + ["az", "bicep", "build", "--file", path, "--stdout"], shell=True + ) + return json.loads(json_template) class Deployer: arm_template = "scaleset_template_windows.bicep" - def __init__(self, + + def __init__( + self, resource_group: str, location: str, subscription_id: Optional[str], - arm_template: str = "scaleset_template_windows.bicep" , + arm_template: str = "scaleset_template_windows.bicep", scaleset_size: int = 1, - ): + ): self.arm_template = arm_template self.resource_group = resource_group self.location = location self.scaleset_size = scaleset_size - self.storage_account = f"{self.resource_group}sa".replace("-", "").replace("_", "") + self.storage_account = f"{self.resource_group}sa".replace("-", "").replace( + "_", "" + ) if subscription_id: self.subscription_id = subscription_id @@ -70,16 +76,11 @@ def __init__(self, profile = get_cli_profile() self.subscription_id = cast(str, profile.get_subscription_id()) - pass - def get_template(path: str) -> Any: - output = bicep_to_arm(path) - return json.loads(output) - def deploy(self): logger.info("deploying") - template = Deployer.get_template(self.arm_template) + template = get_template(self.arm_template) logger.info("deploying 2") credential = AzureCliCredential() client = ResourceManagementClient( @@ -89,10 +90,14 @@ def deploy(self): # client.resource_groups.get(self.resource_group) - storageClient = StorageManagementClient(credential, subscription_id=self.subscription_id) + storageClient = StorageManagementClient( + credential, subscription_id=self.subscription_id + ) try: logger.info("checking for storage account") - prop = storageClient.storage_accounts.get_properties(self.resource_group, self.storage_account) + prop = storageClient.storage_accounts.get_properties( + self.resource_group, self.storage_account + ) logger.info("storage account exists") except HttpResponseError as e: logger.info("storage account does not exist creating a new one") @@ -104,10 +109,11 @@ def deploy(self): allow_blob_public_access=False, minimum_tls_version="TLS1_2", ) - r = storageClient.storage_accounts.begin_create(self.resource_group, self.storage_account, params).result() + r = storageClient.storage_accounts.begin_create( + self.resource_group, self.storage_account, params + ).result() logger.info("storage account created") - file_uris = self.upload_tools(storageClient) logger.info("deploying scaleset") @@ -116,15 +122,13 @@ def deploy(self): ) params = { - "scaleset_name" : { "value": self.resource_group }, - "location" : { "value": self.location }, - "networkSecurityGroups_name" : { "value": "nsg" }, - "adminUsername" : { "value": "onefuzz" }, - "capacity" : { "value": self.scaleset_size }, - "adminPassword": { "value": str(uuid.uuid4()) }, - "file_uris": { "value": file_uris }, - - + "scaleset_name": {"value": self.resource_group}, + "location": {"value": self.location}, + "networkSecurityGroups_name": {"value": "nsg"}, + "adminUsername": {"value": "onefuzz"}, + "capacity": {"value": self.scaleset_size}, + "adminPassword": {"value": str(uuid.uuid4())}, + "file_uris": {"value": file_uris}, } deployment = Deployment( @@ -153,10 +157,16 @@ def upload_tools(self, storageClient: StorageManagementClient) -> List[str]: extracted_path = os.path.join(tmpDir, "tools") onefuzz.tools.get(zip_path) # extract zip - with zipfile.ZipFile(zip_path, 'r') as zip_ref: + with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(extracted_path) - blob_client: BlobServiceClient = BlobServiceClient.from_connection_string(storageClient.storage_accounts.list_keys(self.resource_group, self.storage_account).keys[0].value) + blob_client: BlobServiceClient = BlobServiceClient.from_connection_string( + storageClient.storage_accounts.list_keys( + self.resource_group, self.storage_account + ) + .keys[0] + .value + ) try: container_client = blob_client.create_container("tools") except ResourceExistsError: @@ -167,17 +177,21 @@ def upload_tools(self, storageClient: StorageManagementClient) -> List[str]: for file in os.listdir(extracted_path): logger.debug(f"uploading {file}") - blob_client = container_client.upload_blob(file, os.path.join(extracted_path, file)) - print (f"uploaded {blob_client.url}") - uris.append(blob_client.url) + blob = container_client.upload_blob( + file, os.path.join(extracted_path, file) + ) + print(f"uploaded {blob.url}") + uris.append(blob.url) return uris + def arg_file(arg: str) -> str: if not os.path.isfile(arg): raise argparse.ArgumentTypeError("not a file: %s" % arg) return arg + def main() -> None: formatter = argparse.ArgumentDefaultsHelpFormatter @@ -206,7 +220,9 @@ def main() -> None: logging.basicConfig(level=level) logging.getLogger("deploy").setLevel(logging.INFO) - deployer = Deployer(args.resource_group, args.location, args.subscription_id, args.bicep_template) + deployer = Deployer( + args.resource_group, args.location, args.subscription_id, args.bicep_template + ) deployer.deploy() From 1573f18e4aa48c4f993faf9d60c00354da2c59e7 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 14 Nov 2022 11:17:08 -0800 Subject: [PATCH 17/52] format --- src/deployment/unmanaged/deploy_unamaged_scaleset.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/deployment/unmanaged/deploy_unamaged_scaleset.py b/src/deployment/unmanaged/deploy_unamaged_scaleset.py index 8c271aec17..5e55650ac4 100644 --- a/src/deployment/unmanaged/deploy_unamaged_scaleset.py +++ b/src/deployment/unmanaged/deploy_unamaged_scaleset.py @@ -77,8 +77,7 @@ def __init__( self.subscription_id = cast(str, profile.get_subscription_id()) pass - - def deploy(self): + def deploy(self) -> None: logger.info("deploying") template = get_template(self.arm_template) logger.info("deploying 2") @@ -142,7 +141,7 @@ def deploy(self): ).result() if result.properties.provisioning_state != "Succeeded": - logging.Logger.error( + logger.error( "error deploying: %s", json.dumps(result.as_dict(), indent=4, sort_keys=True), ) From 5295bf7bbfab84b017c52832ebc26c95747c6bde Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 14 Nov 2022 15:49:09 -0800 Subject: [PATCH 18/52] moving stuffs --- src/{deployment => utils}/unmanaged/deploy_unamaged_scaleset.py | 0 src/{deployment => utils}/unmanaged/requirements.txt | 0 src/{deployment => utils}/unmanaged/resource_group.bicep | 0 .../unmanaged/scaleset_template_windows.bicep | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/{deployment => utils}/unmanaged/deploy_unamaged_scaleset.py (100%) rename src/{deployment => utils}/unmanaged/requirements.txt (100%) rename src/{deployment => utils}/unmanaged/resource_group.bicep (100%) rename src/{deployment => utils}/unmanaged/scaleset_template_windows.bicep (100%) diff --git a/src/deployment/unmanaged/deploy_unamaged_scaleset.py b/src/utils/unmanaged/deploy_unamaged_scaleset.py similarity index 100% rename from src/deployment/unmanaged/deploy_unamaged_scaleset.py rename to src/utils/unmanaged/deploy_unamaged_scaleset.py diff --git a/src/deployment/unmanaged/requirements.txt b/src/utils/unmanaged/requirements.txt similarity index 100% rename from src/deployment/unmanaged/requirements.txt rename to src/utils/unmanaged/requirements.txt diff --git a/src/deployment/unmanaged/resource_group.bicep b/src/utils/unmanaged/resource_group.bicep similarity index 100% rename from src/deployment/unmanaged/resource_group.bicep rename to src/utils/unmanaged/resource_group.bicep diff --git a/src/deployment/unmanaged/scaleset_template_windows.bicep b/src/utils/unmanaged/scaleset_template_windows.bicep similarity index 100% rename from src/deployment/unmanaged/scaleset_template_windows.bicep rename to src/utils/unmanaged/scaleset_template_windows.bicep From fd492e7ed851f3f7f00b495717c46782ec47b133 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 15 Nov 2022 17:29:02 -0800 Subject: [PATCH 19/52] fixing bugs --- src/ApiService/ApiService/Functions/Pool.cs | 2 +- .../ApiService/OneFuzzTypes/Model.cs | 4 +- src/pytypes/onefuzztypes/models.py | 2 +- .../unmanaged/deploy_unamaged_scaleset.py | 73 ++++++---- .../unmanaged/scaleset_template_windows.bicep | 130 +++++++++++------- 5 files changed, 132 insertions(+), 79 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Pool.cs b/src/ApiService/ApiService/Functions/Pool.cs index 40160c02b8..830f750a00 100644 --- a/src/ApiService/ApiService/Functions/Pool.cs +++ b/src/ApiService/ApiService/Functions/Pool.cs @@ -134,7 +134,7 @@ private async Task Populate(PoolGetResult p, bool skipSummaries = InstanceId: instanceId, ClientCredentials: null, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, - IsUnamanaged: !p.Managed) + IsUnmanaged: !p.Managed) }; } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 9df19bee25..9e7411b1c0 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -679,12 +679,10 @@ public record AgentConfig( string? MicrosoftTelemetryKey, string? MultiTenantDomain, Guid InstanceId, - bool? IsUnamanaged + bool? IsUnmanaged ); - - public record Vm( string Name, Region Region, diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 0913fcbf61..9b6d0620cb 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -339,7 +339,7 @@ class AgentConfig(BaseModel): microsoft_telemetry_key: Optional[str] multi_tenant_domain: Optional[str] instance_id: UUID - + is_unmanaged: Optional[bool] class TaskUnitConfig(BaseModel): instance_id: UUID diff --git a/src/utils/unmanaged/deploy_unamaged_scaleset.py b/src/utils/unmanaged/deploy_unamaged_scaleset.py index 5e55650ac4..e4ee92741e 100644 --- a/src/utils/unmanaged/deploy_unamaged_scaleset.py +++ b/src/utils/unmanaged/deploy_unamaged_scaleset.py @@ -5,6 +5,7 @@ # the script should reference the deployment files import argparse +import datetime import json import logging import os @@ -17,9 +18,8 @@ from azure.common.credentials import get_cli_profile from azure.core.exceptions import ( - HttpResponseError, + HttpResponseError, # ResourceNotFoundError, ResourceExistsError, - # ResourceNotFoundError, ) from azure.identity import AzureCliCredential from azure.mgmt.resource import ResourceManagementClient @@ -38,8 +38,13 @@ ) from azure.storage.blob import ( BlobServiceClient, + ContainerClient, + generate_container_sas, + ContainerSasPermissions, ) from onefuzz.api import Onefuzz +from onefuzz.azcopy import azcopy_copy +from onefuzztypes.primitives import PoolName logger = logging.getLogger("deploy") @@ -58,9 +63,11 @@ def __init__( self, resource_group: str, location: str, + pool_name: str, subscription_id: Optional[str], arm_template: str = "scaleset_template_windows.bicep", scaleset_size: int = 1, + os: str = "win64", ): self.arm_template = arm_template self.resource_group = resource_group @@ -69,6 +76,9 @@ def __init__( self.storage_account = f"{self.resource_group}sa".replace("-", "").replace( "_", "" ) + self.os = os + self.onefuzz = Onefuzz() + self.pool_name = pool_name if subscription_id: self.subscription_id = subscription_id @@ -77,6 +87,7 @@ def __init__( self.subscription_id = cast(str, profile.get_subscription_id()) pass + def deploy(self) -> None: logger.info("deploying") template = get_template(self.arm_template) @@ -120,19 +131,21 @@ def deploy(self) -> None: self.resource_group, {"location": self.location} ) - params = { + deploy_params = { "scaleset_name": {"value": self.resource_group}, "location": {"value": self.location}, "networkSecurityGroups_name": {"value": "nsg"}, "adminUsername": {"value": "onefuzz"}, "capacity": {"value": self.scaleset_size}, "adminPassword": {"value": str(uuid.uuid4())}, - "file_uris": {"value": file_uris}, + "storageAccountName": {"value": "self.storage_account"}, } deployment = Deployment( properties=DeploymentProperties( - mode=DeploymentMode.incremental, template=template, parameters=params + mode=DeploymentMode.incremental, + template=template, + parameters=deploy_params, ) ) @@ -147,42 +160,51 @@ def deploy(self) -> None: ) sys.exit(1) - def upload_tools(self, storageClient: StorageManagementClient) -> List[str]: + def upload_tools(self, storageClient: StorageManagementClient): logger.info("downloading tools") - onefuzz = Onefuzz() - uris = [] + with tempfile.TemporaryDirectory() as tmpDir: zip_path = os.path.join(tmpDir, "tools.zip") extracted_path = os.path.join(tmpDir, "tools") - onefuzz.tools.get(zip_path) + os.makedirs(extracted_path, exist_ok=True) + self.onefuzz.tools.get(tmpDir) + config = self.onefuzz.pools.get_config(PoolName(self.pool_name)) + + with open(os.path.join(extracted_path, "config.json"), "w") as text_file: + text_file.write(config.json()) + # extract zip with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(extracted_path) - - blob_client: BlobServiceClient = BlobServiceClient.from_connection_string( + account_key = ( storageClient.storage_accounts.list_keys( self.resource_group, self.storage_account ) .keys[0] .value ) + account_url = f"https://{self.storage_account}.blob.core.windows.net" + + blob_client: BlobServiceClient = BlobServiceClient(account_url, account_key) try: - container_client = blob_client.create_container("tools") + container_client: ContainerClient = blob_client.create_container( + "tools" + ) except ResourceExistsError: container_client = blob_client.get_container_client("tools") pass - logger.info("uploading files") - - for file in os.listdir(extracted_path): - logger.debug(f"uploading {file}") - blob = container_client.upload_blob( - file, os.path.join(extracted_path, file) - ) - print(f"uploaded {blob.url}") - uris.append(blob.url) + sas = generate_container_sas( + account_name=container_client.account_name, + container_name=container_client.container_name, + account_key=account_key, + permission=ContainerSasPermissions(read=True, write=True, add=True), + expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1), + ) - return uris + container_sas_url = container_client.url + "/?" + sas + logger.info("uploading files") + azcopy_copy(os.path.join(extracted_path, "*"), container_sas_url) def arg_file(arg: str) -> str: @@ -197,6 +219,7 @@ def main() -> None: parser = argparse.ArgumentParser(formatter_class=formatter) parser.add_argument("location") parser.add_argument("resource_group") + parser.add_argument("pool_name") # parser.add_argument("owner") # parser.add_argument("nsg_config") parser.add_argument( @@ -220,7 +243,11 @@ def main() -> None: logging.getLogger("deploy").setLevel(logging.INFO) deployer = Deployer( - args.resource_group, args.location, args.subscription_id, args.bicep_template + args.resource_group, + args.location, + args.pool_name, + args.subscription_id, + args.bicep_template, ) deployer.deploy() diff --git a/src/utils/unmanaged/scaleset_template_windows.bicep b/src/utils/unmanaged/scaleset_template_windows.bicep index 0a4fa54773..0dabdb56c7 100644 --- a/src/utils/unmanaged/scaleset_template_windows.bicep +++ b/src/utils/unmanaged/scaleset_template_windows.bicep @@ -5,10 +5,26 @@ param adminUsername string = 'onefuzz' param capacity int = 1 param vmSize string = 'Standard_D2s_v3' param tier string = 'Standard' -param fileUris array = [] +param storageAccountName string + @secure() param adminPassword string +resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' existing = { + name: 'storageAccountName' +} + +var StorageBlobDataReader = '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' + +var fileUris = [ + // 'https://${storageAccountName}.blob.${environment().suffixes.storage}/vm-scripts/managed.ps1' + // 'https://${storageAccountName}.blob.${environment().suffixes.storage}/vm-scripts/managed.ps1' + 'https://${storageAccountName}.blob.${environment().suffixes.storage}/tools/config.json' + 'https://${storageAccountName}.blob.${environment().suffixes.storage}/tools/win64/azcopy.exe' + 'https://${storageAccountName}.blob.${environment().suffixes.storage}/tools/win64/setup.ps1' + 'https://${storageAccountName}.blob.${environment().suffixes.storage}/tools/win64/onefuzz.ps1' +] + resource scaleset 'Microsoft.Compute/virtualMachineScaleSets@2022-03-01' = { name: scaleset_name location: location @@ -87,68 +103,67 @@ resource scaleset 'Microsoft.Compute/virtualMachineScaleSets@2022-03-01' = { } extensionProfile: { extensions: [ - { - name: 'OMSExtension' - properties: { - autoUpgradeMinorVersion: true - enableAutomaticUpgrade: false - publisher: 'Microsoft.EnterpriseCloud.Monitoring' - type: 'MicrosoftMonitoringAgent' - typeHandlerVersion: '1.0' - settings: { - workspaceId: '77da01e8-838a-41d9-a6d0-c8b180f68904' - } - } - } - { - name: 'DependencyAgentWindows' - properties: { - autoUpgradeMinorVersion: true - publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' - type: 'DependencyAgentWindows' - typeHandlerVersion: '9.5' - settings: { - } - } - } - { - name: 'Microsoft.Azure.Geneva.GenevaMonitoring' - properties: { - autoUpgradeMinorVersion: true - enableAutomaticUpgrade: true - publisher: 'Microsoft.Azure.Geneva' - type: 'GenevaMonitoring' - typeHandlerVersion: '2.0' - settings: { - } - } - } + // { + // name: 'OMSExtension' + // properties: { + // autoUpgradeMinorVersion: true + // enableAutomaticUpgrade: false + // publisher: 'Microsoft.EnterpriseCloud.Monitoring' + // type: 'MicrosoftMonitoringAgent' + // typeHandlerVersion: '1.0' + // settings: { + // workspaceId: '77da01e8-838a-41d9-a6d0-c8b180f68904' + // } + // } + // } + // { + // name: 'DependencyAgentWindows' + // properties: { + // autoUpgradeMinorVersion: true + // publisher: 'Microsoft.Azure.Monitoring.DependencyAgent' + // type: 'DependencyAgentWindows' + // typeHandlerVersion: '9.5' + // settings: { + // } + // } + // } + // { + // name: 'Microsoft.Azure.Geneva.GenevaMonitoring' + // properties: { + // autoUpgradeMinorVersion: true + // enableAutomaticUpgrade: true + // publisher: 'Microsoft.Azure.Geneva' + // type: 'GenevaMonitoring' + // typeHandlerVersion: '2.0' + // settings: { + // } + // } + // } { name: 'CustomScriptExtension' properties: { autoUpgradeMinorVersion: true - forceUpdateTag: 'e2dc833a-58d2-4f96-a462-4e079f5fc1e0' publisher: 'Microsoft.Compute' type: 'CustomScriptExtension' - typeHandlerVersion: '1.9' + typeHandlerVersion: '1.10' settings: { commandToExecute: 'powershell -ExecutionPolicy Unrestricted -File win64/setup.ps1 -mode fuzz' fileUris: fileUris } } } - { - name: 'Microsoft.Azure.Security.AntimalwareSignature.AntimalwareConfiguration' - properties: { - autoUpgradeMinorVersion: true - enableAutomaticUpgrade: true - publisher: 'Microsoft.Azure.Security.AntimalwareSignature' - type: 'AntimalwareConfiguration' - typeHandlerVersion: '2.0' - settings: { - } - } - } + // { + // name: 'Microsoft.Azure.Security.AntimalwareSignature.AntimalwareConfiguration' + // properties: { + // autoUpgradeMinorVersion: true + // enableAutomaticUpgrade: true + // publisher: 'Microsoft.Azure.Security.AntimalwareSignature' + // type: 'AntimalwareConfiguration' + // typeHandlerVersion: '2.0' + // settings: { + // } + // } + // } ] } priority: 'Regular' @@ -157,6 +172,19 @@ resource scaleset 'Microsoft.Compute/virtualMachineScaleSets@2022-03-01' = { doNotRunExtensionsOnOverprovisionedVMs: false } } +// try to make role assignments to deploy as late as possible in order to have principalId ready +resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { + name: guid('${resourceGroup().id}-user_managed_idenity_read_blob') + properties: { + roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${StorageBlobDataReader}' + principalId: scaleset.identity.principalId + // reference(scalesetIdentity.id, scalesetIdentity.apiVersion, 'Full').properties.principalId + } + dependsOn: [ + storageAccount + ] +} + // resource scaleset_CustomScriptExtension 'Microsoft.Compute/virtualMachineScaleSets/extensions@2022-03-01' = { // parent: scaleset From 71bbffe95f68238108bead684492520375875365 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 15 Nov 2022 19:30:37 -0800 Subject: [PATCH 20/52] formatting --- src/ApiService/ApiService/onefuzzlib/Extension.cs | 2 +- src/pytypes/onefuzztypes/models.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index e1063b7118..8580afc6d5 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -233,7 +233,7 @@ public async Task CreatePoolConfig(Pool pool) { MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, InstanceId: instanceId, - IsUnamanaged: !pool.Managed + IsUnmanaged: !pool.Managed ); return config; } diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 9b6d0620cb..cd00f71678 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -341,6 +341,7 @@ class AgentConfig(BaseModel): instance_id: UUID is_unmanaged: Optional[bool] + class TaskUnitConfig(BaseModel): instance_id: UUID logs: Optional[str] From d0a35a8129aef48f2487d54a09c0a9ba7b800d45 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 15 Nov 2022 22:11:39 -0800 Subject: [PATCH 21/52] changing config name --- src/agent/onefuzz-agent/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index e67ce4d775..3a7b5fadf9 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -42,7 +42,7 @@ pub struct StaticConfig { // Temporary shim type to bridge the current service-provided config. #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct RawStaticConfig { - pub credentials: Option, + pub client_credentials: Option, pub pool_name: String, @@ -66,7 +66,7 @@ impl StaticConfig { pub fn new(data: &[u8]) -> Result { let config: RawStaticConfig = serde_json::from_slice(data)?; - let credentials = match config.credentials { + let credentials = match config.client_credentials { Some(client) => client.into(), None => { // Remove trailing `/`, which is treated as a distinct resource. From 5dce612cae1176a049e2ac4fe095b8f2b3a61c09 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 16 Nov 2022 11:17:06 -0800 Subject: [PATCH 22/52] bug fixes --- src/ApiService/ApiService/UserCredentials.cs | 2 +- .../ApiService/onefuzzlib/EndpointAuthorization.cs | 6 ++++-- src/ApiService/ApiService/onefuzzlib/PoolOperations.cs | 2 +- src/agent/onefuzz/src/auth.rs | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index b1e7179a26..1ac2c5a1bf 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -78,7 +78,7 @@ public virtual async Task> ParseJwtToken(HttpRequest switch (claim.Type) { case "oid": return acc with { UserInfo = acc.UserInfo with { ObjectId = Guid.Parse(claim.Value) } }; - case "appId": + case "appid": return acc with { UserInfo = acc.UserInfo with { ApplicationId = Guid.Parse(claim.Value) } }; case "upn": return acc with { UserInfo = acc.UserInfo with { Upn = claim.Value } }; diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index a566ccb278..3f1be4ab94 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -31,7 +31,7 @@ public class EndpointAuthorization : IEndpointAuthorization { private readonly ILogTracer _log; private readonly GraphServiceClient _graphClient; - private static readonly HashSet AgentRoles = new HashSet { "UnmamagedNode", "ManagedNode" }; + private static readonly HashSet AgentRoles = new HashSet { "UnmanagedNode", "ManagedNode" }; public EndpointAuthorization(IOnefuzzContext context, ILogTracer log, GraphServiceClient graphClient) { _context = context; @@ -202,7 +202,9 @@ public async Async.Task IsAgent(UserAuthInfo authInfo) { } var principalId = await _context.Creds.GetScalesetPrincipalId(); - return principalId == tokenData.ObjectId; + if (principalId == tokenData.ObjectId) { + return true; + } } if (!tokenData.ApplicationId.HasValue) { diff --git a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs index 27867ab470..9170a240b5 100644 --- a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs @@ -89,7 +89,7 @@ public async Task ScheduleWorkset(Pool pool, WorkSet workSet) { } public IAsyncEnumerable GetByClientId(Guid clientId) { - return QueryAsync(filter: TableClient.CreateQueryFilter($"client_id eq {clientId}")); + return QueryAsync(filter: $"client_id eq '{clientId}'"); } public string GetPoolQueue(Guid poolId) diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs index 19a9b08d66..d25a3807f9 100644 --- a/src/agent/onefuzz/src/auth.rs +++ b/src/agent/onefuzz/src/auth.rs @@ -134,7 +134,6 @@ impl ClientCredentials { let response = reqwest::Client::new() .post(url) - .header("Content-Length", "0") .form(&[ ("client_id", self.client_id.to_hyphenated().to_string()), ("client_secret", self.client_secret.expose_ref().to_string()), From f36ddc7532553265b79b44c3d9e0cb2f07249492 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 16 Nov 2022 11:52:58 -0800 Subject: [PATCH 23/52] bug fixes related to the unmanaged nodes - fix the token request in the client - adding a is_unmnaged field to the agentConfig - fix typo in the appId claim type - fix typo in the UnmanagedNode claim value - fix query PoolOperation.GetByClientId --- src/ApiService/ApiService/Functions/Pool.cs | 3 ++- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 5 ++--- src/ApiService/ApiService/UserCredentials.cs | 3 +-- .../onefuzzlib/EndpointAuthorization.cs | 13 +++++++------ .../ApiService/onefuzzlib/PoolOperations.cs | 2 +- src/agent/onefuzz-agent/src/config.rs | 16 +++++++++++++--- src/agent/onefuzz-agent/src/main.rs | 6 +++--- src/agent/onefuzz/src/auth.rs | 1 - src/cli/onefuzz/api.py | 8 +------- src/pytypes/onefuzztypes/models.py | 1 + 10 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Pool.cs b/src/ApiService/ApiService/Functions/Pool.cs index d9fefc49fe..830f750a00 100644 --- a/src/ApiService/ApiService/Functions/Pool.cs +++ b/src/ApiService/ApiService/Functions/Pool.cs @@ -133,7 +133,8 @@ private async Task Populate(PoolGetResult p, bool skipSummaries = HeartbeatQueue: queueSas, InstanceId: instanceId, ClientCredentials: null, - MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain) + MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, + IsUnmanaged: !p.Managed) }; } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index c7620ea38c..9e7411b1c0 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -678,12 +678,11 @@ public record AgentConfig( string? InstanceTelemetryKey, string? MicrosoftTelemetryKey, string? MultiTenantDomain, - Guid InstanceId + Guid InstanceId, + bool? IsUnmanaged ); - - public record Vm( string Name, Region Region, diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index 7446b0fd9b..fb9bf2d7f7 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -77,7 +77,7 @@ public virtual async Task> ParseJwtToken(HttpRequest switch (claim.Type) { case "oid": return acc with { UserInfo = acc.UserInfo with { ObjectId = Guid.Parse(claim.Value) } }; - case "appId": + case "appid": return acc with { UserInfo = acc.UserInfo with { ApplicationId = Guid.Parse(claim.Value) } }; case "upn": return acc with { UserInfo = acc.UserInfo with { Upn = claim.Value } }; @@ -88,7 +88,6 @@ public virtual async Task> ParseJwtToken(HttpRequest return acc; } }); - return OneFuzzResult.Ok(userInfo); } else { var tenantsStr = allowedTenants.OkV is null ? "null" : String.Join(';', allowedTenants.OkV!); diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index 045d43ac9e..6df6eaa51e 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -30,8 +30,7 @@ public class EndpointAuthorization : IEndpointAuthorization { private readonly IOnefuzzContext _context; private readonly ILogTracer _log; private readonly GraphServiceClient _graphClient; - - private static readonly HashSet AgentRoles = new HashSet { "UnmamagedNode", "ManagedNode" }; + private static readonly HashSet AgentRoles = new HashSet { "UnmanagedNode", "ManagedNode" }; public EndpointAuthorization(IOnefuzzContext context, ILogTracer log, GraphServiceClient graphClient) { _context = context; @@ -46,8 +45,8 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu return await _context.RequestHandling.NotOk(req, tokenResult.ErrorV, "token verification", HttpStatusCode.Unauthorized); } - var token = tokenResult.OkV; - if (await IsUser(token)) { + var token = tokenResult.OkV.UserInfo; + if (await IsUser(tokenResult.OkV)) { if (!allowUser) { return await Reject(req, tokenResult.OkV.UserInfo); } @@ -58,7 +57,7 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu } } - if (await IsAgent(token) && !allowAgent) { + if (await IsAgent(tokenResult.OkV) && !allowAgent) { return await Reject(req, tokenResult.OkV.UserInfo); } @@ -201,7 +200,9 @@ public async Async.Task IsAgent(UserAuthInfo authInfo) { } var principalId = await _context.Creds.GetScalesetPrincipalId(); - return principalId == tokenData.ObjectId; + if (principalId == tokenData.ObjectId) { + return true; + } } if (!tokenData.ApplicationId.HasValue) { diff --git a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs index 27867ab470..9170a240b5 100644 --- a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs @@ -89,7 +89,7 @@ public async Task ScheduleWorkset(Pool pool, WorkSet workSet) { } public IAsyncEnumerable GetByClientId(Guid clientId) { - return QueryAsync(filter: TableClient.CreateQueryFilter($"client_id eq {clientId}")); + return QueryAsync(filter: $"client_id eq '{clientId}'"); } public string GetPoolQueue(Guid poolId) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 7d3060e2e5..3a7b5fadf9 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -34,12 +34,15 @@ pub struct StaticConfig { pub heartbeat_queue: Option, pub instance_id: Uuid, + + #[serde(default)] + pub is_unmanaged: bool, } // Temporary shim type to bridge the current service-provided config. #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct RawStaticConfig { - pub credentials: Option, + pub client_credentials: Option, pub pool_name: String, @@ -54,13 +57,16 @@ struct RawStaticConfig { pub heartbeat_queue: Option, pub instance_id: Uuid, + + #[serde(default)] + pub is_unmanaged: bool, } impl StaticConfig { pub fn new(data: &[u8]) -> Result { let config: RawStaticConfig = serde_json::from_slice(data)?; - let credentials = match config.credentials { + let credentials = match config.client_credentials { Some(client) => client.into(), None => { // Remove trailing `/`, which is treated as a distinct resource. @@ -83,6 +89,7 @@ impl StaticConfig { instance_telemetry_key: config.instance_telemetry_key, heartbeat_queue: config.heartbeat_queue, instance_id: config.instance_id, + is_unmanaged: config.is_unmanaged, }; Ok(config) @@ -103,6 +110,7 @@ impl StaticConfig { let multi_tenant_domain = std::env::var("ONEFUZZ_MULTI_TENANT_DOMAIN").ok(); let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; + let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { Some(Url::parse(&key)?) @@ -142,6 +150,7 @@ impl StaticConfig { microsoft_telemetry_key, heartbeat_queue, instance_id, + is_unmanaged, }) } @@ -213,7 +222,8 @@ impl Registration { .append_pair("machine_id", &machine_id.to_string()) .append_pair("machine_name", &machine_name) .append_pair("pool_name", &config.pool_name) - .append_pair("version", env!("ONEFUZZ_VERSION")); + .append_pair("version", env!("ONEFUZZ_VERSION")) + .append_pair("os", std::env::consts::OS); if managed { let scaleset = onefuzz::machine_id::get_scaleset_name().await?; diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 56ff42c7b1..14a8852e78 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -277,10 +277,10 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let registration = match config::Registration::load_existing(config.clone()).await { Ok(registration) => registration, Err(_) => { - if scaleset.is_some() { - config::Registration::create_managed(config.clone()).await? - } else { + if config.is_unmanaged { config::Registration::create_unmanaged(config.clone()).await? + } else { + config::Registration::create_managed(config.clone()).await? } } }; diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs index 19a9b08d66..d25a3807f9 100644 --- a/src/agent/onefuzz/src/auth.rs +++ b/src/agent/onefuzz/src/auth.rs @@ -134,7 +134,6 @@ impl ClientCredentials { let response = reqwest::Client::new() .post(url) - .header("Content-Length", "0") .form(&[ ("client_id", self.client_id.to_hyphenated().to_string()), ("client_secret", self.client_secret.expose_ref().to_string()), diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index b091ded41e..c77e4e3dd6 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1268,13 +1268,7 @@ def get_config(self, pool_name: primitives.PoolName) -> models.AgentConfig: if pool.config is None: raise Exception("Missing AgentConfig in response") - config = pool.config - config.client_credentials = models.ClientCredentials( # nosec - bandit consider this a hard coded password - client_id=pool.client_id, - client_secret="", - ) - - return config + return pool.config def shutdown(self, name: str, *, now: bool = False) -> responses.BoolResult: expanded_name = self._disambiguate( diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 0913fcbf61..cd00f71678 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -339,6 +339,7 @@ class AgentConfig(BaseModel): microsoft_telemetry_key: Optional[str] multi_tenant_domain: Optional[str] instance_id: UUID + is_unmanaged: Optional[bool] class TaskUnitConfig(BaseModel): From 56fc81237b85bf3762392119d55f56ca579da41d Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 16 Nov 2022 12:17:18 -0800 Subject: [PATCH 24/52] remove unused import --- src/ApiService/ApiService/onefuzzlib/Extension.cs | 3 ++- src/ApiService/ApiService/onefuzzlib/PoolOperations.cs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index d9237104bd..dfdace7b93 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -220,7 +220,8 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) { InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey, MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, - InstanceId: instanceId + InstanceId: instanceId, + IsUnmanaged: !pool.Managed ); var fileName = $"{pool.Name}/config.json"; diff --git a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs index 9170a240b5..09c0ec342c 100644 --- a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs @@ -1,6 +1,5 @@ using System.Threading.Tasks; using ApiService.OneFuzzLib.Orm; -using Azure.Data.Tables; namespace Microsoft.OneFuzz.Service; public interface IPoolOperations : IStatefulOrm { From 19b3a3fc63f26cff80ea7b5bac0d7de574820a80 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 16 Nov 2022 13:00:41 -0800 Subject: [PATCH 25/52] build fix --- src/proxy-manager/src/proxy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proxy-manager/src/proxy.rs b/src/proxy-manager/src/proxy.rs index f270cddd30..d0a1fbc36c 100644 --- a/src/proxy-manager/src/proxy.rs +++ b/src/proxy-manager/src/proxy.rs @@ -90,7 +90,7 @@ pub async fn update(data: &ConfigData) -> Result<()> { } let file_name = path.file_name().unwrap().to_string_lossy().to_string(); - if !file_name.starts_with(&PROXY_PREFIX) { + if !file_name.starts_with(PROXY_PREFIX) { continue; } From ea878f517c1617bc6addf32d22a56d179f3b5fe8 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 17 Nov 2022 11:01:24 -0800 Subject: [PATCH 26/52] change unmanaged field to managed --- src/ApiService/ApiService/Functions/Pool.cs | 2 +- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 2 +- .../ApiService/onefuzzlib/Extension.cs | 2 +- src/agent/onefuzz-agent/src/config.rs | 16 ++++++++++------ src/agent/onefuzz-agent/src/main.rs | 6 +++--- src/pytypes/onefuzztypes/models.py | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/ApiService/ApiService/Functions/Pool.cs b/src/ApiService/ApiService/Functions/Pool.cs index 830f750a00..6ca19ec640 100644 --- a/src/ApiService/ApiService/Functions/Pool.cs +++ b/src/ApiService/ApiService/Functions/Pool.cs @@ -134,7 +134,7 @@ private async Task Populate(PoolGetResult p, bool skipSummaries = InstanceId: instanceId, ClientCredentials: null, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, - IsUnmanaged: !p.Managed) + Managed: p.Managed) }; } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 9e7411b1c0..09a1bc4e90 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -679,7 +679,7 @@ public record AgentConfig( string? MicrosoftTelemetryKey, string? MultiTenantDomain, Guid InstanceId, - bool? IsUnmanaged + bool? Managed = true ); diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index dfdace7b93..a9bc508852 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -221,7 +221,7 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) { MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry, MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain, InstanceId: instanceId, - IsUnmanaged: !pool.Managed + Managed: pool.Managed ); var fileName = $"{pool.Name}/config.json"; diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 3a7b5fadf9..e0fafe4841 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -35,8 +35,12 @@ pub struct StaticConfig { pub instance_id: Uuid, - #[serde(default)] - pub is_unmanaged: bool, + #[serde(default = "default_as_true")] + pub managed: bool, +} + +fn default_as_true() -> bool { + true } // Temporary shim type to bridge the current service-provided config. @@ -58,8 +62,8 @@ struct RawStaticConfig { pub instance_id: Uuid, - #[serde(default)] - pub is_unmanaged: bool, + #[serde(default = "default_as_true")] + pub managed: bool, } impl StaticConfig { @@ -89,7 +93,7 @@ impl StaticConfig { instance_telemetry_key: config.instance_telemetry_key, heartbeat_queue: config.heartbeat_queue, instance_id: config.instance_id, - is_unmanaged: config.is_unmanaged, + managed: config.managed, }; Ok(config) @@ -150,7 +154,7 @@ impl StaticConfig { microsoft_telemetry_key, heartbeat_queue, instance_id, - is_unmanaged, + managed: !is_unmanaged, }) } diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 14a8852e78..642e453c98 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -277,10 +277,10 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let registration = match config::Registration::load_existing(config.clone()).await { Ok(registration) => registration, Err(_) => { - if config.is_unmanaged { - config::Registration::create_unmanaged(config.clone()).await? - } else { + if config.managed { config::Registration::create_managed(config.clone()).await? + } else { + config::Registration::create_unmanaged(config.clone()).await? } } }; diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index cd00f71678..858dd18c3d 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -339,7 +339,7 @@ class AgentConfig(BaseModel): microsoft_telemetry_key: Optional[str] multi_tenant_domain: Optional[str] instance_id: UUID - is_unmanaged: Optional[bool] + managed: Optional[bool] = Field(default=True) class TaskUnitConfig(BaseModel): From 4b613275604010f96ac7729f243d4ed4ee393e24 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 17 Nov 2022 12:15:59 -0800 Subject: [PATCH 27/52] build fix --- src/ApiService/ApiService/onefuzzlib/Extension.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index 70712e89b8..6f153e581c 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -236,10 +236,7 @@ public async Task CreatePoolConfig(Pool pool) { Managed: pool.Managed ); - var fileName = $"{pool.Name}/config.json"; - var configJson = JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions()); - await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, fileName, configJson, StorageType.Config); - return await ConfigUrl(WellKnownContainers.VmScripts, fileName, false); + return config; } From 83ec525f3ef0629bcbcad460641e268a6cb010ef Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Fri, 18 Nov 2022 10:04:50 -0800 Subject: [PATCH 28/52] Add managed field to the node entity do not reimage unmanaged nodes --- src/ApiService/ApiService/Functions/AgentRegistration.cs | 4 ++-- src/ApiService/ApiService/OneFuzzTypes/Model.cs | 3 ++- src/ApiService/ApiService/onefuzzlib/NodeOperations.cs | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index 77c8d8e25d..c23fa0b00f 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -156,8 +156,8 @@ private async Async.Task Post(HttpRequestData req) { ScalesetId: scalesetId, InstanceId: instanceId, Version: version, - Os: os ?? pool.Os - + Os: os ?? pool.Os, + Managed: pool.Managed ); var r = await _context.NodeOperations.Replace(node); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 09a1bc4e90..5f9063561a 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -110,7 +110,8 @@ public record Node bool ReimageRequested = false, bool DeleteRequested = false, - bool DebugKeepNode = false + bool DebugKeepNode = false, + bool Managed = true ) : StatefulEntityBase(State) { public List? Tasks { get; set; } diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index 09439d46b1..e01d9e359e 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -341,6 +341,10 @@ public async Async.Task CleanupBusyNodesWithoutWork() { } public async Async.Task ToReimage(Node node, bool done = false) { + if (!node.Managed) { + _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}"); + return node; + } var nodeState = node.State; if (done) { From 054be2c7b07b760b2280bff17f35618016fc293e Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 21 Nov 2022 16:25:45 -0800 Subject: [PATCH 29/52] include a reason when telling a node it is not allowed to schedule work --- .../ApiService/Functions/AgentCanSchedule.cs | 31 +++++++------ .../ApiService/OneFuzzTypes/Responses.cs | 3 +- .../ApiService/onefuzzlib/NodeOperations.cs | 46 +++++++++---------- src/agent/onefuzz-agent/src/agent.rs | 6 ++- src/agent/onefuzz-agent/src/coordinator.rs | 5 +- .../onefuzz-agent/src/coordinator/double.rs | 1 + 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs index 038423b290..63d1d71fe4 100644 --- a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs +++ b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs @@ -42,29 +42,30 @@ private async Async.Task Post(HttpRequestData req) { canScheduleRequest.MachineId.ToString()); } - var allowed = true; - if (!await _context.NodeOperations.CanProcessNewWork(node)) { - allowed = false; - } + + var canProcessNewWork = await _context.NodeOperations.CanProcessNewWork(node); + var allowed = canProcessNewWork.IsAllowed; + var reason = canProcessNewWork.Reason; var task = await _context.TaskOperations.GetByTaskId(canScheduleRequest.TaskId); var workStopped = task == null || task.State.ShuttingDown(); + if (!allowed) { + _log.Info($"Node cannot process new work {node.PoolName:Tag:PoolName} {node.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId} "); + return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped, Reason: reason)); + } + if (workStopped) { _log.Info($"Work stopped for: {canScheduleRequest.MachineId:Tag:MachineId} and {canScheduleRequest.TaskId:Tag:TaskId}"); allowed = false; + reason = "Work stopped"; } - if (allowed) { - var scp = await _context.NodeOperations.AcquireScaleInProtection(node); - if (!scp.IsOk) { - _log.Warning($"Failed to acquire scale in protection for: {node.MachineId:Tag:MachineId} in: {node.PoolName:Tag:PoolName} due to {scp.ErrorV:Tag:Error}"); - } - - _ = scp.OkV; // node could be updated but we don't use it after this - - allowed = scp.IsOk; + var scp = await _context.NodeOperations.AcquireScaleInProtection(node); + if (!scp.IsOk) { + _log.Warning($"Failed to acquire scale in protection for: {node.MachineId:Tag:MachineId} in: {node.PoolName:Tag:PoolName} due to {scp.ErrorV:Tag:Error}"); } - - return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped)); + _ = scp.OkV; // node could be updated but we don't use it after this + allowed = scp.IsOk; + return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped, Reason: reason)); } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index 78fa480402..25e8708c19 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -11,7 +11,8 @@ public static implicit operator BaseResponse(bool value) public record CanSchedule( bool Allowed, - bool WorkStopped + bool WorkStopped, + string? Reason ) : BaseResponse(); public record PendingNodeCommand( diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index 09439d46b1..a56a5ad073 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -6,10 +6,15 @@ namespace Microsoft.OneFuzz.Service; +public record CanProcessNewWorkResponse(bool IsAllowed, string? Reason) { + public static CanProcessNewWorkResponse Allowed() => new CanProcessNewWorkResponse(true, null); + public static CanProcessNewWorkResponse NotAllowed(string reason) => new CanProcessNewWorkResponse(false, reason); +}; + public interface INodeOperations : IStatefulOrm { Task GetByMachineId(Guid machineId); - Task CanProcessNewWork(Node node); + Task CanProcessNewWork(Node node); Task> AcquireScaleInProtection(Node node); Task ReleaseScaleInProtection(Node node); @@ -165,74 +170,65 @@ record NodeInfo(Node Node, Scaleset Scaleset, string InstanceId); return new NodeInfo(node, scalesetResult.OkV, instanceId); } - public async Task CanProcessNewWork(Node node) { + + + public async Task CanProcessNewWork(Node node) { if (IsOutdated(node) && _context.ServiceConfiguration.OneFuzzAllowOutdatedAgent != "true") { - _logTracer.Info($"can_process_new_work agent and service versions differ, stopping node. {node.MachineId:Tag:MachineId} {node.Version:Tag:AgentVersion} {_context.ServiceConfiguration.OneFuzzVersion:Tag:ServiceVersion}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("agent and service versions differ"); } if (IsTooOld(node)) { - _logTracer.Info($"can_process_new_work node is too old {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is too old"); } if (!node.State.CanProcessNewWork()) { - _logTracer.Info($"can_process_new_work node not in appropriate state for new work {node.MachineId:Tag:MachineId} {node.State:Tag:State}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("node not in appropriate state for new work"); } if (node.State.ReadyForReset()) { - _logTracer.Info($"can_process_new_work node is set for reset {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set for reset"); } if (node.DeleteRequested) { - _logTracer.Info($"can_process_new_work is set to be deleted {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set to be deleted"); } if (node.ReimageRequested) { - _logTracer.Info($"can_process_new_work is set to be reimaged {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set to be reimaged"); } if (await CouldShrinkScaleset(node)) { - _logTracer.Info($"can_process_new_work node scheduled to shrink {node.MachineId:Tag:MachineId}"); _ = await SetHalt(node); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is scheduled to shrink"); } if (node.ScalesetId != null) { var scalesetResult = await _context.ScalesetOperations.GetById(node.ScalesetId.Value); if (!scalesetResult.IsOk) { - _logTracer.Info($"can_process_new_work invalid scaleset {node.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("invalid scaleset"); } var scaleset = scalesetResult.OkV; if (!scaleset.State.IsAvailable()) { - _logTracer.Info($"can_process_new_work scaleset not available for work {scaleset.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId} {scaleset.State:Tag:State}"); - return false; + return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'"); ; } } var poolResult = await _context.PoolOperations.GetByName(node.PoolName); if (!poolResult.IsOk) { - _logTracer.Info($"can_schedule - invalid pool {node.PoolName:Tag:PoolName} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("invalid pool"); ; } var pool = poolResult.OkV; if (!PoolStateHelper.Available.Contains(pool.State)) { - _logTracer.Info($"can_schedule - pool is not available for work {node.PoolName:Tag:PoolName} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("pool is not available for work"); ; } - return true; + return CanProcessNewWorkResponse.Allowed(); } diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index ab0358ed65..d52e224bc5 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -159,10 +159,14 @@ impl Agent { } } } else { + let reason = can_schedule.reason.map_or("".to_string(), |r| r); // We cannot schedule the work set. Depending on why, we want to either drop the work // (because it is no longer valid for anyone) or do nothing (because our version is out // of date, and we want another node to pick it up). - warn!("unable to schedule work set: {:?}", msg.work_set); + warn!( + "unable to schedule work set: {:?}, Reason {}", + msg.work_set, reason + ); // If `work_stopped`, the work set is not valid for any node, and we should drop it for the // entire pool by claiming but not executing it. diff --git a/src/agent/onefuzz-agent/src/coordinator.rs b/src/agent/onefuzz-agent/src/coordinator.rs index 273dacb1f2..fb466a1808 100644 --- a/src/agent/onefuzz-agent/src/coordinator.rs +++ b/src/agent/onefuzz-agent/src/coordinator.rs @@ -125,7 +125,7 @@ pub struct CanScheduleRequest { task_id: Uuid, } -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CanSchedule { /// If true, then the receiving node can schedule the work. /// Otherwise, the receiver should inspect `work_stopped`. @@ -137,6 +137,9 @@ pub struct CanSchedule { /// No node in the pool may schedule the work, so the receiving node should /// claim (delete) and drop the work set. pub work_stopped: bool, + + /// contains the reason why the work was stopped or not allowed. + pub reason: Option, } #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/src/agent/onefuzz-agent/src/coordinator/double.rs b/src/agent/onefuzz-agent/src/coordinator/double.rs index 9e4ac9f644..6abbe77140 100644 --- a/src/agent/onefuzz-agent/src/coordinator/double.rs +++ b/src/agent/onefuzz-agent/src/coordinator/double.rs @@ -24,6 +24,7 @@ impl ICoordinator for CoordinatorDouble { Ok(CanSchedule { allowed: true, work_stopped: true, + reason: None, }) } } From 569266e3e45617fd34babd844f9972a3ec932fb9 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 21 Nov 2022 16:25:45 -0800 Subject: [PATCH 30/52] include a reason when telling a node it is not allowed to schedule work --- .../ApiService/Functions/AgentCanSchedule.cs | 31 +++++++------ .../ApiService/OneFuzzTypes/Responses.cs | 3 +- .../ApiService/onefuzzlib/NodeOperations.cs | 46 +++++++++---------- src/agent/onefuzz-agent/src/agent.rs | 6 ++- src/agent/onefuzz-agent/src/coordinator.rs | 5 +- .../onefuzz-agent/src/coordinator/double.rs | 1 + 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs index 038423b290..63d1d71fe4 100644 --- a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs +++ b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs @@ -42,29 +42,30 @@ private async Async.Task Post(HttpRequestData req) { canScheduleRequest.MachineId.ToString()); } - var allowed = true; - if (!await _context.NodeOperations.CanProcessNewWork(node)) { - allowed = false; - } + + var canProcessNewWork = await _context.NodeOperations.CanProcessNewWork(node); + var allowed = canProcessNewWork.IsAllowed; + var reason = canProcessNewWork.Reason; var task = await _context.TaskOperations.GetByTaskId(canScheduleRequest.TaskId); var workStopped = task == null || task.State.ShuttingDown(); + if (!allowed) { + _log.Info($"Node cannot process new work {node.PoolName:Tag:PoolName} {node.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId} "); + return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped, Reason: reason)); + } + if (workStopped) { _log.Info($"Work stopped for: {canScheduleRequest.MachineId:Tag:MachineId} and {canScheduleRequest.TaskId:Tag:TaskId}"); allowed = false; + reason = "Work stopped"; } - if (allowed) { - var scp = await _context.NodeOperations.AcquireScaleInProtection(node); - if (!scp.IsOk) { - _log.Warning($"Failed to acquire scale in protection for: {node.MachineId:Tag:MachineId} in: {node.PoolName:Tag:PoolName} due to {scp.ErrorV:Tag:Error}"); - } - - _ = scp.OkV; // node could be updated but we don't use it after this - - allowed = scp.IsOk; + var scp = await _context.NodeOperations.AcquireScaleInProtection(node); + if (!scp.IsOk) { + _log.Warning($"Failed to acquire scale in protection for: {node.MachineId:Tag:MachineId} in: {node.PoolName:Tag:PoolName} due to {scp.ErrorV:Tag:Error}"); } - - return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped)); + _ = scp.OkV; // node could be updated but we don't use it after this + allowed = scp.IsOk; + return await RequestHandling.Ok(req, new CanSchedule(Allowed: allowed, WorkStopped: workStopped, Reason: reason)); } } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index 09bf4608bd..a09be7d061 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -11,7 +11,8 @@ public static implicit operator BaseResponse(bool value) public record CanSchedule( bool Allowed, - bool WorkStopped + bool WorkStopped, + string? Reason ) : BaseResponse(); public record PendingNodeCommand( diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index e01d9e359e..181772bc81 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -6,10 +6,15 @@ namespace Microsoft.OneFuzz.Service; +public record CanProcessNewWorkResponse(bool IsAllowed, string? Reason) { + public static CanProcessNewWorkResponse Allowed() => new CanProcessNewWorkResponse(true, null); + public static CanProcessNewWorkResponse NotAllowed(string reason) => new CanProcessNewWorkResponse(false, reason); +}; + public interface INodeOperations : IStatefulOrm { Task GetByMachineId(Guid machineId); - Task CanProcessNewWork(Node node); + Task CanProcessNewWork(Node node); Task> AcquireScaleInProtection(Node node); Task ReleaseScaleInProtection(Node node); @@ -165,74 +170,65 @@ record NodeInfo(Node Node, Scaleset Scaleset, string InstanceId); return new NodeInfo(node, scalesetResult.OkV, instanceId); } - public async Task CanProcessNewWork(Node node) { + + + public async Task CanProcessNewWork(Node node) { if (IsOutdated(node) && _context.ServiceConfiguration.OneFuzzAllowOutdatedAgent != "true") { - _logTracer.Info($"can_process_new_work agent and service versions differ, stopping node. {node.MachineId:Tag:MachineId} {node.Version:Tag:AgentVersion} {_context.ServiceConfiguration.OneFuzzVersion:Tag:ServiceVersion}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("agent and service versions differ"); } if (IsTooOld(node)) { - _logTracer.Info($"can_process_new_work node is too old {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is too old"); } if (!node.State.CanProcessNewWork()) { - _logTracer.Info($"can_process_new_work node not in appropriate state for new work {node.MachineId:Tag:MachineId} {node.State:Tag:State}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("node not in appropriate state for new work"); } if (node.State.ReadyForReset()) { - _logTracer.Info($"can_process_new_work node is set for reset {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set for reset"); } if (node.DeleteRequested) { - _logTracer.Info($"can_process_new_work is set to be deleted {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set to be deleted"); } if (node.ReimageRequested) { - _logTracer.Info($"can_process_new_work is set to be reimaged {node.MachineId:Tag:MachineId}"); _ = await Stop(node, done: true); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is set to be reimaged"); } if (await CouldShrinkScaleset(node)) { - _logTracer.Info($"can_process_new_work node scheduled to shrink {node.MachineId:Tag:MachineId}"); _ = await SetHalt(node); - return false; + return CanProcessNewWorkResponse.NotAllowed("node is scheduled to shrink"); } if (node.ScalesetId != null) { var scalesetResult = await _context.ScalesetOperations.GetById(node.ScalesetId.Value); if (!scalesetResult.IsOk) { - _logTracer.Info($"can_process_new_work invalid scaleset {node.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("invalid scaleset"); } var scaleset = scalesetResult.OkV; if (!scaleset.State.IsAvailable()) { - _logTracer.Info($"can_process_new_work scaleset not available for work {scaleset.ScalesetId:Tag:ScalesetId} - {node.MachineId:Tag:MachineId} {scaleset.State:Tag:State}"); - return false; + return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'"); ; } } var poolResult = await _context.PoolOperations.GetByName(node.PoolName); if (!poolResult.IsOk) { - _logTracer.Info($"can_schedule - invalid pool {node.PoolName:Tag:PoolName} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("invalid pool"); ; } var pool = poolResult.OkV; if (!PoolStateHelper.Available.Contains(pool.State)) { - _logTracer.Info($"can_schedule - pool is not available for work {node.PoolName:Tag:PoolName} - {node.MachineId:Tag:MachineId}"); - return false; + return CanProcessNewWorkResponse.NotAllowed("pool is not available for work"); ; } - return true; + return CanProcessNewWorkResponse.Allowed(); } diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index ab0358ed65..d52e224bc5 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -159,10 +159,14 @@ impl Agent { } } } else { + let reason = can_schedule.reason.map_or("".to_string(), |r| r); // We cannot schedule the work set. Depending on why, we want to either drop the work // (because it is no longer valid for anyone) or do nothing (because our version is out // of date, and we want another node to pick it up). - warn!("unable to schedule work set: {:?}", msg.work_set); + warn!( + "unable to schedule work set: {:?}, Reason {}", + msg.work_set, reason + ); // If `work_stopped`, the work set is not valid for any node, and we should drop it for the // entire pool by claiming but not executing it. diff --git a/src/agent/onefuzz-agent/src/coordinator.rs b/src/agent/onefuzz-agent/src/coordinator.rs index 273dacb1f2..fb466a1808 100644 --- a/src/agent/onefuzz-agent/src/coordinator.rs +++ b/src/agent/onefuzz-agent/src/coordinator.rs @@ -125,7 +125,7 @@ pub struct CanScheduleRequest { task_id: Uuid, } -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CanSchedule { /// If true, then the receiving node can schedule the work. /// Otherwise, the receiver should inspect `work_stopped`. @@ -137,6 +137,9 @@ pub struct CanSchedule { /// No node in the pool may schedule the work, so the receiving node should /// claim (delete) and drop the work set. pub work_stopped: bool, + + /// contains the reason why the work was stopped or not allowed. + pub reason: Option, } #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/src/agent/onefuzz-agent/src/coordinator/double.rs b/src/agent/onefuzz-agent/src/coordinator/double.rs index 9e4ac9f644..6abbe77140 100644 --- a/src/agent/onefuzz-agent/src/coordinator/double.rs +++ b/src/agent/onefuzz-agent/src/coordinator/double.rs @@ -24,6 +24,7 @@ impl ICoordinator for CoordinatorDouble { Ok(CanSchedule { allowed: true, work_stopped: true, + reason: None, }) } } From 2cb3d447c1fac19ee9e0b7ba5efa748e9149de22 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 21 Nov 2022 17:43:28 -0800 Subject: [PATCH 31/52] format --- src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs | 2 +- src/ApiService/ApiService/onefuzzlib/NodeOperations.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index 5e14c57a58..b302819b5b 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -59,7 +59,7 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu if (await IsAgent(tokenResult.OkV) && !allowAgent) { - return await Reject(req, token); + return await Reject(req, token); } return await method(req); diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index 181772bc81..79e52bde5b 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -339,7 +339,7 @@ public async Async.Task CleanupBusyNodesWithoutWork() { public async Async.Task ToReimage(Node node, bool done = false) { if (!node.Managed) { _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}"); - return node; + return node; } var nodeState = node.State; From 379553eb33cdbdeb95b0421165054e079cea78cd Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 21 Nov 2022 20:35:51 -0800 Subject: [PATCH 32/52] making machine identity a parameter --- src/agent/onefuzz-agent/src/agent.rs | 6 +- src/agent/onefuzz-agent/src/commands.rs | 7 +- src/agent/onefuzz-agent/src/config.rs | 32 ++- src/agent/onefuzz-agent/src/heartbeat.rs | 9 +- src/agent/onefuzz-agent/src/main.rs | 23 +- src/agent/onefuzz-agent/src/scheduler.rs | 8 +- src/agent/onefuzz-agent/src/worker.rs | 19 +- .../src/local/libfuzzer_test_input.rs | 1 + .../onefuzz-task/src/local/test_input.rs | 1 + src/agent/onefuzz-task/src/managed/cmd.rs | 3 +- .../src/tasks/analysis/generic.rs | 2 +- src/agent/onefuzz-task/src/tasks/config.rs | 22 +- .../onefuzz-task/src/tasks/coverage/dotnet.rs | 4 +- .../src/tasks/coverage/generic.rs | 2 +- .../onefuzz-task/src/tasks/fuzz/generator.rs | 3 +- .../src/tasks/fuzz/libfuzzer/dotnet.rs | 1 + .../src/tasks/fuzz/libfuzzer/generic.rs | 1 + .../onefuzz-task/src/tasks/fuzz/supervisor.rs | 4 +- src/agent/onefuzz-task/src/tasks/heartbeat.rs | 5 +- .../onefuzz-task/src/tasks/merge/generic.rs | 2 +- .../src/tasks/merge/libfuzzer_merge.rs | 2 + .../src/tasks/regression/generic.rs | 1 + .../src/tasks/regression/libfuzzer.rs | 1 + .../src/tasks/report/dotnet/generic.rs | 4 +- .../onefuzz-task/src/tasks/report/generic.rs | 7 +- .../src/tasks/report/libfuzzer_report.rs | 8 +- src/agent/onefuzz/src/expand.rs | 46 ++-- src/agent/onefuzz/src/input_tester.rs | 6 +- src/agent/onefuzz/src/libfuzzer.rs | 22 +- src/agent/onefuzz/src/machine_id.rs | 244 ++++++++++-------- 30 files changed, 311 insertions(+), 185 deletions(-) diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index d52e224bc5..ad3be468c1 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -26,6 +26,7 @@ pub struct Agent { heartbeat: Option, previous_state: NodeState, last_poll_command: Result, PollCommandError>, + managed: bool, } impl Agent { @@ -37,6 +38,7 @@ impl Agent { work_queue: Box, worker_runner: Box, heartbeat: Option, + managed: bool, ) -> Self { let scheduler = Some(scheduler); let previous_state = NodeState::Init; @@ -52,6 +54,7 @@ impl Agent { heartbeat, previous_state, last_poll_command, + managed, } } @@ -290,7 +293,8 @@ impl Agent { Ok(None) => {} Ok(Some(cmd)) => { info!("agent received node command: {:?}", cmd); - self.scheduler()?.execute_command(cmd).await?; + let managed = self.managed; + self.scheduler()?.execute_command(cmd, managed).await?; } Err(PollCommandError::RequestFailed(err)) => { // If we failed to request commands, this could be the service diff --git a/src/agent/onefuzz-agent/src/commands.rs b/src/agent/onefuzz-agent/src/commands.rs index 049ccc595d..92e3fe3791 100644 --- a/src/agent/onefuzz-agent/src/commands.rs +++ b/src/agent/onefuzz-agent/src/commands.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use anyhow::{Context, Result}; -use onefuzz::{auth::Secret, machine_id::get_scaleset_name}; +use onefuzz::auth::Secret; use std::process::Stdio; use tokio::{fs, io::AsyncWriteExt, process::Command}; @@ -32,11 +32,6 @@ pub struct SshKeyInfo { #[cfg(target_family = "windows")] pub async fn add_ssh_key(key_info: &SshKeyInfo) -> Result<()> { - if get_scaleset_name().await?.is_none() { - warn!("adding ssh keys only supported on managed nodes"); - return Ok(()); - } - let mut ssh_path = PathBuf::from(env::var("ProgramData").unwrap_or_else(|_| "c:\\programdata".to_string())); ssh_path.push("ssh"); diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index e0fafe4841..5d5f65a9ea 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -6,6 +6,7 @@ use onefuzz::{ auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}, http::{is_auth_error_code, ResponseExt}, jitter::delay_with_jitter, + machine_id::MachineIdentity, }; use onefuzz_telemetry::{InstanceTelemetryKey, MicrosoftTelemetryKey}; use reqwest_retry::SendRetry; @@ -37,6 +38,8 @@ pub struct StaticConfig { #[serde(default = "default_as_true")] pub managed: bool, + + pub machine_identity: MachineIdentity, } fn default_as_true() -> bool { @@ -64,10 +67,12 @@ struct RawStaticConfig { #[serde(default = "default_as_true")] pub managed: bool, + + pub machine_identity: Option, } impl StaticConfig { - pub fn new(data: &[u8]) -> Result { + pub async fn new(data: &[u8]) -> Result { let config: RawStaticConfig = serde_json::from_slice(data)?; let credentials = match config.client_credentials { @@ -84,6 +89,11 @@ impl StaticConfig { managed.into() } }; + let machine_identity = match config.machine_identity { + Some(machine_identity) => machine_identity, + None => MachineIdentity::from_metadata().await?, + }; + let config = StaticConfig { credentials, pool_name: config.pool_name, @@ -94,16 +104,17 @@ impl StaticConfig { heartbeat_queue: config.heartbeat_queue, instance_id: config.instance_id, managed: config.managed, + machine_identity, }; Ok(config) } - pub fn from_file(config_path: impl AsRef) -> Result { + pub async fn from_file(config_path: impl AsRef) -> Result { let config_path = config_path.as_ref(); let data = std::fs::read(config_path) .with_context(|| format!("unable to read config file: {}", config_path.display()))?; - Self::new(&data) + Self::new(&data).await } pub fn from_env() -> Result { @@ -115,6 +126,7 @@ impl StaticConfig { let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); + let machine_identity = MachineIdentity::from_env()?; let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { Some(Url::parse(&key)?) @@ -155,6 +167,7 @@ impl StaticConfig { heartbeat_queue, instance_id, managed: !is_unmanaged, + machine_identity, }) } @@ -218,22 +231,21 @@ const REGISTRATION_RETRY_PERIOD: Duration = Duration::from_secs(60); impl Registration { pub async fn create(config: StaticConfig, managed: bool, timeout: Duration) -> Result { let token = config.credentials.access_token().await?; - let machine_name = onefuzz::machine_id::get_machine_name().await?; - let machine_id = onefuzz::machine_id::get_machine_id().await?; + let machine_name = &config.machine_identity.machine_name; + let machine_id = config.machine_identity.machine_id; let mut url = config.register_url(); url.query_pairs_mut() .append_pair("machine_id", &machine_id.to_string()) - .append_pair("machine_name", &machine_name) + .append_pair("machine_name", machine_name) .append_pair("pool_name", &config.pool_name) .append_pair("version", env!("ONEFUZZ_VERSION")) .append_pair("os", std::env::consts::OS); if managed { - let scaleset = onefuzz::machine_id::get_scaleset_name().await?; - match scaleset { + match &config.machine_identity.scaleset_name { Some(scaleset) => { - url.query_pairs_mut().append_pair("scaleset_id", &scaleset); + url.query_pairs_mut().append_pair("scaleset_id", scaleset); } None => { anyhow::bail!("managed instance without scaleset name"); @@ -284,7 +296,7 @@ impl Registration { pub async fn load_existing(config: StaticConfig) -> Result { let dynamic_config = DynamicConfig::load().await?; - let machine_id = onefuzz::machine_id::get_machine_id().await?; + let machine_id = config.machine_identity.machine_id; let mut registration = Self { config, dynamic_config, diff --git a/src/agent/onefuzz-agent/src/heartbeat.rs b/src/agent/onefuzz-agent/src/heartbeat.rs index 166489cde6..3e6415ec06 100644 --- a/src/agent/onefuzz-agent/src/heartbeat.rs +++ b/src/agent/onefuzz-agent/src/heartbeat.rs @@ -2,7 +2,6 @@ // Licensed under the MIT License. use crate::onefuzz::heartbeat::HeartbeatClient; -use crate::onefuzz::machine_id::{get_machine_id, get_machine_name}; use anyhow::Result; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -29,9 +28,11 @@ pub struct AgentContext { pub type AgentHeartbeatClient = HeartbeatClient; -pub async fn init_agent_heartbeat(queue_url: Url) -> Result { - let node_id = get_machine_id().await?; - let machine_name = get_machine_name().await?; +pub async fn init_agent_heartbeat( + queue_url: Url, + node_id: Uuid, + machine_name: String, +) -> Result { let hb = HeartbeatClient::init_heartbeat( AgentContext { node_id, diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 642e453c98..ddb549d072 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -21,10 +21,7 @@ use std::process::{Command, Stdio}; use anyhow::{Context, Result}; use clap::Parser; -use onefuzz::{ - machine_id::{get_machine_id, get_scaleset_name}, - process::ExitStatus, -}; +use onefuzz::process::ExitStatus; use onefuzz_telemetry::{self as telemetry, EventData, Role}; use std::io::{self, Write}; use uuid::Uuid; @@ -202,7 +199,7 @@ async fn load_config(opt: RunOpt) -> Result { info!("loading supervisor agent config"); let config = match &opt.config_path { - Some(config_path) => StaticConfig::from_file(config_path)?, + Some(config_path) => StaticConfig::from_file(config_path).await?, None => StaticConfig::from_env()?, }; @@ -266,11 +263,11 @@ async fn check_existing_worksets(coordinator: &mut coordinator::Coordinator) -> async fn run_agent(config: StaticConfig) -> Result<()> { telemetry::set_property(EventData::InstanceId(config.instance_id)); - telemetry::set_property(EventData::MachineId(get_machine_id().await?)); + telemetry::set_property(EventData::MachineId(config.machine_identity.machine_id)); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); telemetry::set_property(EventData::Role(Role::Supervisor)); - let scaleset = get_scaleset_name().await?; - if let Some(scaleset_name) = &scaleset { + + if let Some(scaleset_name) = &config.machine_identity.scaleset_name { telemetry::set_property(EventData::ScalesetId(scaleset_name.to_string())); } @@ -300,7 +297,14 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let work_queue = work::WorkQueue::new(registration.clone())?; let agent_heartbeat = match config.heartbeat_queue { - Some(url) => Some(init_agent_heartbeat(url).await?), + Some(url) => Some( + init_agent_heartbeat( + url, + config.machine_identity.machine_id, + config.machine_identity.machine_name, + ) + .await?, + ), None => None, }; let mut agent = agent::Agent::new( @@ -311,6 +315,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { Box::new(work_queue), Box::new(worker::WorkerRunner), agent_heartbeat, + config.managed, ); info!("running agent"); diff --git a/src/agent/onefuzz-agent/src/scheduler.rs b/src/agent/onefuzz-agent/src/scheduler.rs index 77bf38936a..e9b5fcc8ae 100644 --- a/src/agent/onefuzz-agent/src/scheduler.rs +++ b/src/agent/onefuzz-agent/src/scheduler.rs @@ -54,10 +54,14 @@ impl Scheduler { Self::default() } - pub async fn execute_command(&mut self, cmd: &NodeCommand) -> Result<()> { + pub async fn execute_command(&mut self, cmd: &NodeCommand, managed: bool) -> Result<()> { match cmd { NodeCommand::AddSshKey(ssh_key_info) => { - add_ssh_key(ssh_key_info).await?; + if managed { + add_ssh_key(ssh_key_info).await?; + } else { + warn!("adding ssh keys only supported on managed nodes"); + } } NodeCommand::StopTask(stop_task) => { if let Scheduler::Busy(state) = self { diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index c0d154e7a6..52b6daecff 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. use std::{ + collections::HashMap, path::{Path, PathBuf}, process::{Child, ChildStderr, ChildStdout, Command, Stdio}, thread::{self, JoinHandle}, @@ -8,12 +9,17 @@ use std::{ use anyhow::{format_err, Context as AnyhowContext, Result}; use downcast_rs::Downcast; -use onefuzz::process::{ExitStatus, Output}; +use onefuzz::{ + machine_id::MachineIdentity, + process::{ExitStatus, Output}, +}; use tokio::fs; use crate::buffer::TailBuffer; use crate::work::*; +use serde_json::Value; + // Max length of captured output streams from worker child processes. const MAX_TAIL_LEN: usize = 40960; @@ -209,9 +215,18 @@ impl IWorkerRunner for WorkerRunner { debug!("created worker working dir: {}", working_dir.display()); + // inject the machine_identity in the config file + let work_config = work.config.expose_ref(); + let mut config: HashMap = serde_json::from_str(work_config.as_str())?; + + config.insert( + "machine_identity".to_string(), + serde_json::to_value(MachineIdentity::default())?, + ); + let config_path = work.config_path()?; - fs::write(&config_path, work.config.expose_ref()) + fs::write(&config_path, serde_json::to_string(&config)?.as_bytes()) .await .with_context(|| format!("unable to save task config: {}", config_path.display()))?; diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs index 5d800f0f36..690fa66a52 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs @@ -35,6 +35,7 @@ pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option, event_sender: Option) -> Result<()> { let check_oom = out_of_memory(min_available_memory_bytes); let common = config.common().clone(); - let machine_id = get_machine_id().await?; + let machine_id = common.machine_identity.machine_id; let task_logger = if let Some(logs) = common.logs.clone() { let rx = onefuzz_telemetry::subscribe_to_events()?; diff --git a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs index aa5e3e80ef..9717cae3e7 100644 --- a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs @@ -198,7 +198,7 @@ pub async fn run_tool( let target_exe = try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .input_path(&input) diff --git a/src/agent/onefuzz-task/src/tasks/config.rs b/src/agent/onefuzz-task/src/tasks/config.rs index a128b2fe84..1a31c39e86 100644 --- a/src/agent/onefuzz-task/src/tasks/config.rs +++ b/src/agent/onefuzz-task/src/tasks/config.rs @@ -10,7 +10,7 @@ use crate::tasks::{ merge, regression, report, }; use anyhow::Result; -use onefuzz::machine_id::{get_machine_id, get_scaleset_name}; +use onefuzz::machine_id::MachineIdentity; use onefuzz_telemetry::{ self as telemetry, Event::task_start, EventData, InstanceTelemetryKey, MicrosoftTelemetryKey, Role, @@ -58,6 +58,8 @@ pub struct CommonConfig { /// Can be disabled by setting to 0. #[serde(default = "default_min_available_memory_mb")] pub min_available_memory_mb: u64, + + pub machine_identity: MachineIdentity, } impl CommonConfig { @@ -67,8 +69,15 @@ impl CommonConfig { ) -> Result> { match &self.heartbeat_queue { Some(url) => { - let hb = init_task_heartbeat(url.clone(), self.task_id, self.job_id, initial_delay) - .await?; + let hb = init_task_heartbeat( + url.clone(), + self.task_id, + self.job_id, + initial_delay, + self.machine_identity.machine_id, + self.machine_identity.machine_name.clone(), + ) + .await?; Ok(Some(hb)) } None => Ok(None), @@ -215,13 +224,14 @@ impl Config { pub async fn run(self) -> Result<()> { telemetry::set_property(EventData::JobId(self.common().job_id)); telemetry::set_property(EventData::TaskId(self.common().task_id)); - telemetry::set_property(EventData::MachineId(get_machine_id().await?)); + telemetry::set_property(EventData::MachineId( + self.common().machine_identity.machine_id, + )); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); telemetry::set_property(EventData::InstanceId(self.common().instance_id)); telemetry::set_property(EventData::Role(Role::Agent)); - let scaleset = get_scaleset_name().await?; - if let Some(scaleset_name) = &scaleset { + if let Some(scaleset_name) = &self.common().machine_identity.scaleset_name { telemetry::set_property(EventData::ScalesetId(scaleset_name.to_string())); } diff --git a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs index 6e0df226aa..d4f3c72c81 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs @@ -266,7 +266,7 @@ impl<'a> TaskContext<'a> { // Try to expand `target_exe` with support for `{tools_dir}`. // // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. - let expand = Expand::new().tools_dir(tools_dir); + let expand = Expand::new(&self.config.common.machine_identity).tools_dir(tools_dir); let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; let expanded_path = Path::new(&expanded); @@ -293,7 +293,7 @@ impl<'a> TaskContext<'a> { async fn command_for_input(&self, input: &Path) -> Result { let target_exe = self.target_exe().await?; - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .input_path(input) diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 2acc4458a3..535ee5660e 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -288,7 +288,7 @@ impl<'a> TaskContext<'a> { try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) .await?; - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .input_path(input) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs index 36e55eb82e..a9dad3e48c 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs @@ -102,6 +102,7 @@ impl GeneratorTask { &target_exe, &self.config.target_options, &self.config.target_env, + self.config.common.machine_identity.clone(), ) .check_asan_log(self.config.check_asan_log) .check_debugger(self.config.check_debugger) @@ -163,7 +164,7 @@ impl GeneratorTask { ) -> Result<()> { utils::reset_tmp_dir(&output_dir).await?; let (mut generator, generator_path) = { - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .setup_dir(&self.config.common.setup_dir) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs index ba08e22842..9bf36eb5a0 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs @@ -86,6 +86,7 @@ impl common::LibFuzzerType for LibFuzzerDotnet { options, env, &config.common.setup_dir, + config.common.machine_identity.clone(), )) } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs index 51da454f0d..1d9d118d9f 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs @@ -28,6 +28,7 @@ impl common::LibFuzzerType for GenericLibFuzzer { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), )) } } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs index 05955afcd7..df71249c09 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs @@ -144,7 +144,7 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { let monitor_path = if let Some(stats_file) = &config.stats_file { Some( - Expand::new() + Expand::new(&config.common.machine_identity) .machine_id() .await? .runtime_dir(runtime_dir.path()) @@ -204,7 +204,7 @@ async fn start_supervisor( None }; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .supervisor_exe(&config.supervisor_exe) diff --git a/src/agent/onefuzz-task/src/tasks/heartbeat.rs b/src/agent/onefuzz-task/src/tasks/heartbeat.rs index a82ee5b649..515fa39d0c 100644 --- a/src/agent/onefuzz-task/src/tasks/heartbeat.rs +++ b/src/agent/onefuzz-task/src/tasks/heartbeat.rs @@ -2,7 +2,6 @@ // Licensed under the MIT License. use crate::onefuzz::heartbeat::HeartbeatClient; -use crate::onefuzz::machine_id::{get_machine_id, get_machine_name}; use anyhow::Result; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -40,9 +39,9 @@ pub async fn init_task_heartbeat( task_id: Uuid, job_id: Uuid, initial_delay: Option, + machine_id: Uuid, + machine_name: String, ) -> Result { - let machine_id = get_machine_id().await?; - let machine_name = get_machine_name().await?; let hb = HeartbeatClient::init_heartbeat( TaskContext { task_id, diff --git a/src/agent/onefuzz-task/src/tasks/merge/generic.rs b/src/agent/onefuzz-task/src/tasks/merge/generic.rs index b1b6265093..de804154cf 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/generic.rs @@ -130,7 +130,7 @@ async fn merge(config: &Config, output_dir: impl AsRef) -> Result<()> { let target_exe = try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .input_marker(&config.supervisor_input_marker) diff --git a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs index b119452207..7b8c6f7b43 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs @@ -46,6 +46,7 @@ pub async fn spawn(config: Arc) -> Result<()> { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), ); fuzzer.verify(config.check_fuzzer_help, None).await?; @@ -159,6 +160,7 @@ pub async fn merge_inputs( config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), ); merger .merge(&config.unique_inputs.local_path, &candidates) diff --git a/src/agent/onefuzz-task/src/tasks/regression/generic.rs b/src/agent/onefuzz-task/src/tasks/regression/generic.rs index cbc1e29965..68d0f1643c 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/generic.rs @@ -74,6 +74,7 @@ impl RegressionHandler for GenericRegressionTask { check_asan_log: self.config.check_asan_log, check_debugger: self.config.check_debugger, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; generic::test_input(args).await } diff --git a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs index ed0322cc04..5a5ce3990b 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs @@ -71,6 +71,7 @@ impl RegressionHandler for LibFuzzerRegressionTask { target_timeout: self.config.target_timeout, check_retry_count: self.config.check_retry_count, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; libfuzzer_report::test_input(args).await } diff --git a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs index a119f303c4..894d1615ee 100644 --- a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs @@ -132,7 +132,7 @@ impl AsanProcessor { // Try to expand `target_exe` with support for `{tools_dir}`. // // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. - let expand = Expand::new().tools_dir(tools_dir); + let expand = Expand::new(&self.config.common.machine_identity).tools_dir(tools_dir); let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; let expanded_path = Path::new(&expanded); @@ -180,7 +180,7 @@ impl AsanProcessor { let mut args = vec![target_exe]; args.extend(self.config.target_options.clone()); - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .input_path(input) .setup_dir(&self.config.common.setup_dir); let expanded_args = expand.evaluate(&args)?; diff --git a/src/agent/onefuzz-task/src/tasks/report/generic.rs b/src/agent/onefuzz-task/src/tasks/report/generic.rs index 490f4e8a8e..2b3c02c955 100644 --- a/src/agent/onefuzz-task/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/generic.rs @@ -10,7 +10,9 @@ use crate::tasks::{ }; use anyhow::{Context, Result}; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, input_tester::Tester, sha256, syncdir::SyncedDir}; +use onefuzz::{ + blob::BlobUrl, input_tester::Tester, machine_id::MachineIdentity, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -118,6 +120,7 @@ pub struct TestInputArgs<'a> { pub check_asan_log: bool, pub check_debugger: bool, pub minimized_stack_depth: Option, + pub machine_identity: MachineIdentity, } pub async fn test_input(args: TestInputArgs<'_>) -> Result { @@ -126,6 +129,7 @@ pub async fn test_input(args: TestInputArgs<'_>) -> Result { args.target_exe, args.target_options, args.target_env, + args.machine_identity.clone(), ) .check_asan_log(args.check_asan_log) .check_debugger(args.check_debugger) @@ -211,6 +215,7 @@ impl<'a> GenericReportProcessor<'a> { check_asan_log: self.config.check_asan_log, check_debugger: self.config.check_debugger, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; test_input(args).await.context("test input failed") } diff --git a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs index 47e51b0a0d..58d9612179 100644 --- a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs @@ -10,7 +10,9 @@ use crate::tasks::{ }; use anyhow::{Context, Result}; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir}; +use onefuzz::{ + blob::BlobUrl, libfuzzer::LibFuzzer, machine_id::MachineIdentity, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -74,6 +76,7 @@ impl ReportTask { self.config.target_options.clone(), self.config.target_env.clone(), &self.config.common.setup_dir, + self.config.common.machine_identity.clone(), ); fuzzer.verify(self.config.check_fuzzer_help, None).await } @@ -120,6 +123,7 @@ pub struct TestInputArgs<'a> { pub target_timeout: Option, pub check_retry_count: u64, pub minimized_stack_depth: Option, + pub machine_identity: MachineIdentity, } pub async fn test_input(args: TestInputArgs<'_>) -> Result { @@ -128,6 +132,7 @@ pub async fn test_input(args: TestInputArgs<'_>) -> Result { args.target_options.to_vec(), args.target_env.clone(), args.setup_dir, + args.machine_identity, ); let task_id = args.task_id; @@ -215,6 +220,7 @@ impl AsanProcessor { target_timeout: self.config.target_timeout, check_retry_count: self.config.check_retry_count, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; let result = test_input(args).await?; diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index d0986d615e..0317aa282b 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{machine_id::get_machine_id, sha256::digest_file_blocking}; +use crate::{machine_id::MachineIdentity, sha256::digest_file_blocking}; use anyhow::{format_err, Context, Result}; use onefuzz_telemetry::{InstanceTelemetryKey, MicrosoftTelemetryKey}; use std::path::{Path, PathBuf}; @@ -89,16 +89,11 @@ impl PlaceHolder { pub struct Expand<'a> { values: HashMap>, -} - -impl Default for Expand<'_> { - fn default() -> Self { - Self::new() - } + machine_identity: &'a MachineIdentity, } impl<'a> Expand<'a> { - pub fn new() -> Self { + pub fn new(machine_identity: &'a MachineIdentity) -> Self { let mut values = HashMap::new(); values.insert( PlaceHolder::InputFileNameNoExt.get_string(), @@ -113,12 +108,15 @@ impl<'a> Expand<'a> { ExpandedValue::Mapping(Box::new(Expand::input_file_sha256)), ); - Self { values } + Self { + values, + machine_identity, + } } // Must be manually called to enable the use of async library code. pub async fn machine_id(self) -> Result> { - let id = get_machine_id().await?; + let id = self.machine_identity.machine_id; let value = id.to_string(); Ok(self.set_value(PlaceHolder::MachineId, ExpandedValue::Scalar(value))) } @@ -171,7 +169,10 @@ impl<'a> Expand<'a> { pub fn set_value(self, name: PlaceHolder, value: ExpandedValue<'a>) -> Self { let mut values = self.values; values.insert(name.get_string(), value); - Self { values } + Self { + values, + machine_identity: self.machine_identity, + } } pub fn set_optional_ref<'l, T: 'l>( @@ -412,6 +413,8 @@ impl<'a> Expand<'a> { #[cfg(test)] mod tests { + use crate::machine_id::MachineIdentity; + use super::Expand; use anyhow::{Context, Result}; use std::path::Path; @@ -421,7 +424,7 @@ mod tests { fn test_expand_nested() -> Result<()> { let supervisor_options = vec!["{target_options}".to_string()]; let target_options: Vec<_> = vec!["a", "b", "c"].iter().map(|p| p.to_string()).collect(); - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .target_options(&target_options) .evaluate(&supervisor_options)?; let expected = vec!["a b c"]; @@ -464,7 +467,7 @@ mod tests { let input_corpus_dir = "src"; let generated_inputs_dir = "src"; - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .input_corpus(Path::new(input_corpus_dir)) .generated_inputs(Path::new(generated_inputs_dir)) .target_options(&my_options) @@ -498,14 +501,16 @@ mod tests { ] ); - assert!(Expand::new().evaluate(&my_args).is_err()); + assert!(Expand::new(&MachineIdentity::default()) + .evaluate(&my_args) + .is_err()); Ok(()) } #[test] fn test_expand_in_string() -> Result<()> { - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .input_path("src/lib.rs") .evaluate_value("a {input} b")?; assert!(result.contains("lib.rs")); @@ -514,10 +519,17 @@ mod tests { #[tokio::test] async fn test_expand_machine_id() -> Result<()> { - let expand = Expand::new().machine_id().await?; + let machine_id = Uuid::new_v4(); + let expand = Expand::new(&MachineIdentity { + machine_id: machine_id, + ..Default::default() + }) + .machine_id() + .await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. - Uuid::parse_str(&expanded)?; + let expanded_machine_id = Uuid::parse_str(&expanded)?; + assert_eq!(expanded_machine_id, machine_id); Ok(()) } } diff --git a/src/agent/onefuzz/src/input_tester.rs b/src/agent/onefuzz/src/input_tester.rs index 6eb9e0a655..ec4375455d 100644 --- a/src/agent/onefuzz/src/input_tester.rs +++ b/src/agent/onefuzz/src/input_tester.rs @@ -7,6 +7,7 @@ use crate::{ asan::{add_asan_log_env, check_asan_path, check_asan_string}, env::{get_path_with_directory, update_path, LD_LIBRARY_PATH, PATH}, expand::Expand, + machine_id::MachineIdentity, process::run_cmd, }; use anyhow::{Context, Error, Result}; @@ -36,6 +37,7 @@ pub struct Tester<'a> { check_retry_count: u64, add_setup_to_ld_library_path: bool, add_setup_to_path: bool, + machine_identity: MachineIdentity, } #[derive(Debug)] @@ -57,6 +59,7 @@ impl<'a> Tester<'a> { exe_path: &'a Path, arguments: &'a [String], environ: &'a HashMap, + machine_identity: MachineIdentity, ) -> Self { Self { setup_dir, @@ -70,6 +73,7 @@ impl<'a> Tester<'a> { check_retry_count: 0, add_setup_to_ld_library_path: false, add_setup_to_path: false, + machine_identity, } } @@ -289,7 +293,7 @@ impl<'a> Tester<'a> { }; let (argv, env) = { - let expand = Expand::new() + let expand = Expand::new(&self.machine_identity) .machine_id() .await? .input_path(input_file) diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index d221732639..43eb4a4fae 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -6,6 +6,7 @@ use crate::{ expand::Expand, fs::{list_files, write_file}, input_tester::{TestResult, Tester}, + machine_id::MachineIdentity, }; use anyhow::{Context, Result}; use rand::seq::SliceRandom; @@ -39,6 +40,7 @@ pub struct LibFuzzer { exe: PathBuf, options: Vec, env: HashMap, + machine_identity: MachineIdentity, } impl LibFuzzer { @@ -47,12 +49,14 @@ impl LibFuzzer { options: Vec, env: HashMap, setup_dir: impl Into, + machine_identity: MachineIdentity, ) -> Self { Self { exe: exe.into(), options, env, setup_dir: setup_dir.into(), + machine_identity, } } @@ -98,7 +102,7 @@ impl LibFuzzer { ); } - let expand = Expand::new() + let expand = Expand::new(&self.machine_identity) .machine_id() .await? .target_exe(&self.exe) @@ -310,11 +314,17 @@ impl LibFuzzer { let mut options = self.options.clone(); options.push("{input}".to_string()); - let mut tester = Tester::new(&self.setup_dir, &self.exe, &options, &self.env) - .check_asan_stderr(true) - .check_retry_count(retry) - .add_setup_to_path(true) - .set_optional(timeout, |tester, timeout| tester.timeout(timeout)); + let mut tester = Tester::new( + &self.setup_dir, + &self.exe, + &options, + &self.env, + self.machine_identity.clone(), + ) + .check_asan_stderr(true) + .check_retry_count(retry) + .add_setup_to_path(true) + .set_optional(timeout, |tester, timeout| tester.timeout(timeout)); if cfg!(target_family = "unix") { tester = tester.add_setup_to_ld_library_path(true); diff --git a/src/agent/onefuzz/src/machine_id.rs b/src/agent/onefuzz/src/machine_id.rs index 33b222776d..d608a95e3d 100644 --- a/src/agent/onefuzz/src/machine_id.rs +++ b/src/agent/onefuzz/src/machine_id.rs @@ -10,6 +10,13 @@ use std::time::Duration; use tokio::fs; use uuid::Uuid; +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Default, Serialize)] +pub struct MachineIdentity { + pub machine_id: Uuid, + pub machine_name: String, + pub scaleset_name: Option, +} + // https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service#tracking-vm-running-on-azure const IMS_ID_URL: &str = "http://169.254.169.254/metadata/instance/compute/vmId?api-version=2020-06-01&format=text"; @@ -19,127 +26,156 @@ const VM_NAME_URL: &str = "http://169.254.169.254/metadata/instance/compute/name?api-version=2020-06-01&format=text"; const VM_SCALESET_NAME: &str = - "http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2020-06-01&format=text"; - -pub async fn get_ims_id() -> Result { - let path = onefuzz_etc()?.join("ims_id"); - let body = match fs::read_to_string(&path).await { - Ok(body) => body, - Err(_) => { - let resp = reqwest::Client::new() - .get(IMS_ID_URL) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - .context("get_ims_id")?; - let body = resp.text().await?; - write_file(path, &body).await?; - body - } - }; - - let value = Uuid::parse_str(&body)?; - Ok(value) -} +"http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2020-06-01&format=text"; + +const COMPUTE_METADATA_URL: &str = + "http://169.254.169.254/metadata/instance/compute?api-version=2020-06-01"; + +impl MachineIdentity { + pub async fn from_metadata() -> Result { + let machine_id = Self::get_machine_id().await?; + let machine_name = Self::get_machine_name().await?; + let scaleset_name = Self::get_scaleset_name().await?; + + Ok(Self { + machine_id, + machine_name, + scaleset_name, + }) + } -pub async fn get_machine_name() -> Result { - let path = onefuzz_etc()?.join("machine_name"); - let body = match fs::read_to_string(&path).await { - Ok(body) => body, - Err(_) => { - let resp = reqwest::Client::new() - .get(VM_NAME_URL) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - .context("get_machine_name")?; - let body = resp.text().await?; - write_file(path, &body).await?; - body - } - }; + pub fn from_env() -> Result { + let machine_id = Uuid::parse_str(&std::env::var("ONEFUZZ_MACHINE_ID")?)?; + let machine_name = std::env::var("ONEFUZZ_MACHINE_NAME")?; + let scaleset_name = std::env::var("ONEFUZZ_SCALESET_NAME").ok(); - Ok(body) -} + Ok(Self { + machine_id, + machine_name, + scaleset_name, + }) + } -pub async fn get_scaleset_name() -> Result> { - let path = onefuzz_etc()?.join("scaleset_name"); - if let Ok(scaleset_name) = fs::read_to_string(&path).await { - return Ok(Some(scaleset_name)); + pub async fn get_ims_id() -> Result { + let path = onefuzz_etc()?.join("ims_id"); + let body = match fs::read_to_string(&path).await { + Ok(body) => body, + Err(_) => { + let resp = reqwest::Client::new() + .get(IMS_ID_URL) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + .context("get_ims_id")?; + let body = resp.text().await?; + write_file(path, &body).await?; + body + } + }; + + let value = Uuid::parse_str(&body)?; + Ok(value) } - if let Ok(resp) = reqwest::Client::new() - .get(VM_SCALESET_NAME) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - { - let body = resp.text().await?; - write_file(path, &body).await?; - Ok(Some(body)) - } else { - Ok(None) + pub async fn get_machine_name() -> Result { + let path = onefuzz_etc()?.join("machine_name"); + let body = match fs::read_to_string(&path).await { + Ok(body) => body, + Err(_) => { + let resp = reqwest::Client::new() + .get(VM_NAME_URL) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + .context("get_machine_name")?; + let body = resp.text().await?; + write_file(path, &body).await?; + body + } + }; + + Ok(body) } -} -#[cfg(target_os = "linux")] -pub async fn get_os_machine_id() -> Result { - let path = Path::new("/etc/machine-id"); - let contents = fs::read_to_string(&path) - .await - .with_context(|| format!("unable to read machine_id: {}", path.display()))?; - let uuid = Uuid::parse_str(contents.trim())?; - Ok(uuid) -} + pub async fn get_scaleset_name() -> Result> { + let path = onefuzz_etc()?.join("scaleset_name"); + if let Ok(scaleset_name) = fs::read_to_string(&path).await { + return Ok(Some(scaleset_name)); + } -#[cfg(target_os = "windows")] -pub async fn get_os_machine_id() -> Result { - use winreg::enums::{HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_64KEY}; - use winreg::RegKey; - - let key: &str = "SOFTWARE\\Microsoft\\Cryptography"; - - let hklm = RegKey::predef(HKEY_LOCAL_MACHINE); - let crypt = if let Ok(crypt) = hklm.open_subkey_with_flags(key, KEY_READ) { - crypt - } else { - hklm.open_subkey_with_flags(key, KEY_READ | KEY_WOW64_64KEY)? - }; - let guid: String = crypt.get_value("MachineGuid")?; - Ok(Uuid::parse_str(&guid)?) -} + if let Ok(resp) = reqwest::Client::new() + .get(VM_SCALESET_NAME) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + { + let body = resp.text().await?; + write_file(path, &body).await?; + Ok(Some(body)) + } else { + Ok(None) + } + } -async fn get_machine_id_impl() -> Result { - let ims_id = get_ims_id().await; - if ims_id.is_ok() { - return ims_id; + #[cfg(target_os = "linux")] + pub async fn get_os_machine_id() -> Result { + let path = Path::new("/etc/machine-id"); + let contents = fs::read_to_string(&path) + .await + .with_context(|| format!("unable to read machine_id: {}", path.display()))?; + let uuid = Uuid::parse_str(contents.trim())?; + Ok(uuid) } - let machine_id = get_os_machine_id().await; - if machine_id.is_ok() { - return machine_id; + #[cfg(target_os = "windows")] + pub async fn get_os_machine_id() -> Result { + use winreg::enums::{HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_64KEY}; + use winreg::RegKey; + + let key: &str = "SOFTWARE\\Microsoft\\Cryptography"; + + let hklm = RegKey::predef(HKEY_LOCAL_MACHINE); + let crypt = if let Ok(crypt) = hklm.open_subkey_with_flags(key, KEY_READ) { + crypt + } else { + hklm.open_subkey_with_flags(key, KEY_READ | KEY_WOW64_64KEY)? + }; + let guid: String = crypt.get_value("MachineGuid")?; + Ok(Uuid::parse_str(&guid)?) } - Ok(Uuid::new_v4()) -} + async fn get_machine_id_impl() -> Result { + let ims_id = Self::get_ims_id().await; + if ims_id.is_ok() { + return ims_id; + } -pub async fn get_machine_id() -> Result { - let path = onefuzz_etc()?.join("machine_id"); - let result = match fs::read_to_string(&path).await { - Ok(body) => Uuid::parse_str(&body)?, - Err(_) => { - let value = get_machine_id_impl().await?; - write_file(path, &value.to_string()).await?; - value + let machine_id = Self::get_os_machine_id().await; + if machine_id.is_ok() { + return machine_id; } - }; - Ok(result) + + Ok(Uuid::new_v4()) + } + + pub async fn get_machine_id() -> Result { + let path = onefuzz_etc()?.join("machine_id"); + let result = match fs::read_to_string(&path).await { + Ok(body) => Uuid::parse_str(&body)?, + Err(_) => { + let value = Self::get_machine_id_impl().await?; + write_file(path, &value.to_string()).await?; + value + } + }; + Ok(result) + } } #[tokio::test] async fn test_get_machine_id() { - get_os_machine_id().await.unwrap(); + MachineIdentity::get_os_machine_id().await.unwrap(); } From 88f24eca4184c22ba4ac979032ceaec9fde3991a Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 21 Nov 2022 20:35:51 -0800 Subject: [PATCH 33/52] making machine identity a parameter --- src/agent/onefuzz-agent/src/agent.rs | 6 +- src/agent/onefuzz-agent/src/commands.rs | 7 +- src/agent/onefuzz-agent/src/config.rs | 32 ++- src/agent/onefuzz-agent/src/heartbeat.rs | 9 +- src/agent/onefuzz-agent/src/main.rs | 23 +- src/agent/onefuzz-agent/src/scheduler.rs | 8 +- src/agent/onefuzz-agent/src/worker.rs | 19 +- .../src/local/libfuzzer_test_input.rs | 1 + .../onefuzz-task/src/local/test_input.rs | 1 + src/agent/onefuzz-task/src/managed/cmd.rs | 3 +- .../src/tasks/analysis/generic.rs | 2 +- src/agent/onefuzz-task/src/tasks/config.rs | 22 +- .../onefuzz-task/src/tasks/coverage/dotnet.rs | 4 +- .../src/tasks/coverage/generic.rs | 2 +- .../onefuzz-task/src/tasks/fuzz/generator.rs | 3 +- .../src/tasks/fuzz/libfuzzer/dotnet.rs | 1 + .../src/tasks/fuzz/libfuzzer/generic.rs | 1 + .../onefuzz-task/src/tasks/fuzz/supervisor.rs | 4 +- src/agent/onefuzz-task/src/tasks/heartbeat.rs | 5 +- .../onefuzz-task/src/tasks/merge/generic.rs | 2 +- .../src/tasks/merge/libfuzzer_merge.rs | 2 + .../src/tasks/regression/generic.rs | 1 + .../src/tasks/regression/libfuzzer.rs | 1 + .../src/tasks/report/dotnet/generic.rs | 4 +- .../onefuzz-task/src/tasks/report/generic.rs | 7 +- .../src/tasks/report/libfuzzer_report.rs | 8 +- src/agent/onefuzz/src/expand.rs | 46 ++-- src/agent/onefuzz/src/input_tester.rs | 6 +- src/agent/onefuzz/src/libfuzzer.rs | 22 +- src/agent/onefuzz/src/machine_id.rs | 244 ++++++++++-------- 30 files changed, 311 insertions(+), 185 deletions(-) diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index ab0358ed65..1531471fe3 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -26,6 +26,7 @@ pub struct Agent { heartbeat: Option, previous_state: NodeState, last_poll_command: Result, PollCommandError>, + managed: bool, } impl Agent { @@ -37,6 +38,7 @@ impl Agent { work_queue: Box, worker_runner: Box, heartbeat: Option, + managed: bool, ) -> Self { let scheduler = Some(scheduler); let previous_state = NodeState::Init; @@ -52,6 +54,7 @@ impl Agent { heartbeat, previous_state, last_poll_command, + managed, } } @@ -286,7 +289,8 @@ impl Agent { Ok(None) => {} Ok(Some(cmd)) => { info!("agent received node command: {:?}", cmd); - self.scheduler()?.execute_command(cmd).await?; + let managed = self.managed; + self.scheduler()?.execute_command(cmd, managed).await?; } Err(PollCommandError::RequestFailed(err)) => { // If we failed to request commands, this could be the service diff --git a/src/agent/onefuzz-agent/src/commands.rs b/src/agent/onefuzz-agent/src/commands.rs index 049ccc595d..92e3fe3791 100644 --- a/src/agent/onefuzz-agent/src/commands.rs +++ b/src/agent/onefuzz-agent/src/commands.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use anyhow::{Context, Result}; -use onefuzz::{auth::Secret, machine_id::get_scaleset_name}; +use onefuzz::auth::Secret; use std::process::Stdio; use tokio::{fs, io::AsyncWriteExt, process::Command}; @@ -32,11 +32,6 @@ pub struct SshKeyInfo { #[cfg(target_family = "windows")] pub async fn add_ssh_key(key_info: &SshKeyInfo) -> Result<()> { - if get_scaleset_name().await?.is_none() { - warn!("adding ssh keys only supported on managed nodes"); - return Ok(()); - } - let mut ssh_path = PathBuf::from(env::var("ProgramData").unwrap_or_else(|_| "c:\\programdata".to_string())); ssh_path.push("ssh"); diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index e0fafe4841..5d5f65a9ea 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -6,6 +6,7 @@ use onefuzz::{ auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}, http::{is_auth_error_code, ResponseExt}, jitter::delay_with_jitter, + machine_id::MachineIdentity, }; use onefuzz_telemetry::{InstanceTelemetryKey, MicrosoftTelemetryKey}; use reqwest_retry::SendRetry; @@ -37,6 +38,8 @@ pub struct StaticConfig { #[serde(default = "default_as_true")] pub managed: bool, + + pub machine_identity: MachineIdentity, } fn default_as_true() -> bool { @@ -64,10 +67,12 @@ struct RawStaticConfig { #[serde(default = "default_as_true")] pub managed: bool, + + pub machine_identity: Option, } impl StaticConfig { - pub fn new(data: &[u8]) -> Result { + pub async fn new(data: &[u8]) -> Result { let config: RawStaticConfig = serde_json::from_slice(data)?; let credentials = match config.client_credentials { @@ -84,6 +89,11 @@ impl StaticConfig { managed.into() } }; + let machine_identity = match config.machine_identity { + Some(machine_identity) => machine_identity, + None => MachineIdentity::from_metadata().await?, + }; + let config = StaticConfig { credentials, pool_name: config.pool_name, @@ -94,16 +104,17 @@ impl StaticConfig { heartbeat_queue: config.heartbeat_queue, instance_id: config.instance_id, managed: config.managed, + machine_identity, }; Ok(config) } - pub fn from_file(config_path: impl AsRef) -> Result { + pub async fn from_file(config_path: impl AsRef) -> Result { let config_path = config_path.as_ref(); let data = std::fs::read(config_path) .with_context(|| format!("unable to read config file: {}", config_path.display()))?; - Self::new(&data) + Self::new(&data).await } pub fn from_env() -> Result { @@ -115,6 +126,7 @@ impl StaticConfig { let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; let is_unmanaged = std::env::var("ONEFUZZ_IS_UNMANAGED").is_ok(); + let machine_identity = MachineIdentity::from_env()?; let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { Some(Url::parse(&key)?) @@ -155,6 +167,7 @@ impl StaticConfig { heartbeat_queue, instance_id, managed: !is_unmanaged, + machine_identity, }) } @@ -218,22 +231,21 @@ const REGISTRATION_RETRY_PERIOD: Duration = Duration::from_secs(60); impl Registration { pub async fn create(config: StaticConfig, managed: bool, timeout: Duration) -> Result { let token = config.credentials.access_token().await?; - let machine_name = onefuzz::machine_id::get_machine_name().await?; - let machine_id = onefuzz::machine_id::get_machine_id().await?; + let machine_name = &config.machine_identity.machine_name; + let machine_id = config.machine_identity.machine_id; let mut url = config.register_url(); url.query_pairs_mut() .append_pair("machine_id", &machine_id.to_string()) - .append_pair("machine_name", &machine_name) + .append_pair("machine_name", machine_name) .append_pair("pool_name", &config.pool_name) .append_pair("version", env!("ONEFUZZ_VERSION")) .append_pair("os", std::env::consts::OS); if managed { - let scaleset = onefuzz::machine_id::get_scaleset_name().await?; - match scaleset { + match &config.machine_identity.scaleset_name { Some(scaleset) => { - url.query_pairs_mut().append_pair("scaleset_id", &scaleset); + url.query_pairs_mut().append_pair("scaleset_id", scaleset); } None => { anyhow::bail!("managed instance without scaleset name"); @@ -284,7 +296,7 @@ impl Registration { pub async fn load_existing(config: StaticConfig) -> Result { let dynamic_config = DynamicConfig::load().await?; - let machine_id = onefuzz::machine_id::get_machine_id().await?; + let machine_id = config.machine_identity.machine_id; let mut registration = Self { config, dynamic_config, diff --git a/src/agent/onefuzz-agent/src/heartbeat.rs b/src/agent/onefuzz-agent/src/heartbeat.rs index 166489cde6..3e6415ec06 100644 --- a/src/agent/onefuzz-agent/src/heartbeat.rs +++ b/src/agent/onefuzz-agent/src/heartbeat.rs @@ -2,7 +2,6 @@ // Licensed under the MIT License. use crate::onefuzz::heartbeat::HeartbeatClient; -use crate::onefuzz::machine_id::{get_machine_id, get_machine_name}; use anyhow::Result; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -29,9 +28,11 @@ pub struct AgentContext { pub type AgentHeartbeatClient = HeartbeatClient; -pub async fn init_agent_heartbeat(queue_url: Url) -> Result { - let node_id = get_machine_id().await?; - let machine_name = get_machine_name().await?; +pub async fn init_agent_heartbeat( + queue_url: Url, + node_id: Uuid, + machine_name: String, +) -> Result { let hb = HeartbeatClient::init_heartbeat( AgentContext { node_id, diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 642e453c98..ddb549d072 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -21,10 +21,7 @@ use std::process::{Command, Stdio}; use anyhow::{Context, Result}; use clap::Parser; -use onefuzz::{ - machine_id::{get_machine_id, get_scaleset_name}, - process::ExitStatus, -}; +use onefuzz::process::ExitStatus; use onefuzz_telemetry::{self as telemetry, EventData, Role}; use std::io::{self, Write}; use uuid::Uuid; @@ -202,7 +199,7 @@ async fn load_config(opt: RunOpt) -> Result { info!("loading supervisor agent config"); let config = match &opt.config_path { - Some(config_path) => StaticConfig::from_file(config_path)?, + Some(config_path) => StaticConfig::from_file(config_path).await?, None => StaticConfig::from_env()?, }; @@ -266,11 +263,11 @@ async fn check_existing_worksets(coordinator: &mut coordinator::Coordinator) -> async fn run_agent(config: StaticConfig) -> Result<()> { telemetry::set_property(EventData::InstanceId(config.instance_id)); - telemetry::set_property(EventData::MachineId(get_machine_id().await?)); + telemetry::set_property(EventData::MachineId(config.machine_identity.machine_id)); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); telemetry::set_property(EventData::Role(Role::Supervisor)); - let scaleset = get_scaleset_name().await?; - if let Some(scaleset_name) = &scaleset { + + if let Some(scaleset_name) = &config.machine_identity.scaleset_name { telemetry::set_property(EventData::ScalesetId(scaleset_name.to_string())); } @@ -300,7 +297,14 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let work_queue = work::WorkQueue::new(registration.clone())?; let agent_heartbeat = match config.heartbeat_queue { - Some(url) => Some(init_agent_heartbeat(url).await?), + Some(url) => Some( + init_agent_heartbeat( + url, + config.machine_identity.machine_id, + config.machine_identity.machine_name, + ) + .await?, + ), None => None, }; let mut agent = agent::Agent::new( @@ -311,6 +315,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { Box::new(work_queue), Box::new(worker::WorkerRunner), agent_heartbeat, + config.managed, ); info!("running agent"); diff --git a/src/agent/onefuzz-agent/src/scheduler.rs b/src/agent/onefuzz-agent/src/scheduler.rs index 77bf38936a..e9b5fcc8ae 100644 --- a/src/agent/onefuzz-agent/src/scheduler.rs +++ b/src/agent/onefuzz-agent/src/scheduler.rs @@ -54,10 +54,14 @@ impl Scheduler { Self::default() } - pub async fn execute_command(&mut self, cmd: &NodeCommand) -> Result<()> { + pub async fn execute_command(&mut self, cmd: &NodeCommand, managed: bool) -> Result<()> { match cmd { NodeCommand::AddSshKey(ssh_key_info) => { - add_ssh_key(ssh_key_info).await?; + if managed { + add_ssh_key(ssh_key_info).await?; + } else { + warn!("adding ssh keys only supported on managed nodes"); + } } NodeCommand::StopTask(stop_task) => { if let Scheduler::Busy(state) = self { diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index c0d154e7a6..52b6daecff 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. use std::{ + collections::HashMap, path::{Path, PathBuf}, process::{Child, ChildStderr, ChildStdout, Command, Stdio}, thread::{self, JoinHandle}, @@ -8,12 +9,17 @@ use std::{ use anyhow::{format_err, Context as AnyhowContext, Result}; use downcast_rs::Downcast; -use onefuzz::process::{ExitStatus, Output}; +use onefuzz::{ + machine_id::MachineIdentity, + process::{ExitStatus, Output}, +}; use tokio::fs; use crate::buffer::TailBuffer; use crate::work::*; +use serde_json::Value; + // Max length of captured output streams from worker child processes. const MAX_TAIL_LEN: usize = 40960; @@ -209,9 +215,18 @@ impl IWorkerRunner for WorkerRunner { debug!("created worker working dir: {}", working_dir.display()); + // inject the machine_identity in the config file + let work_config = work.config.expose_ref(); + let mut config: HashMap = serde_json::from_str(work_config.as_str())?; + + config.insert( + "machine_identity".to_string(), + serde_json::to_value(MachineIdentity::default())?, + ); + let config_path = work.config_path()?; - fs::write(&config_path, work.config.expose_ref()) + fs::write(&config_path, serde_json::to_string(&config)?.as_bytes()) .await .with_context(|| format!("unable to save task config: {}", config_path.display()))?; diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs index 5d800f0f36..690fa66a52 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs @@ -35,6 +35,7 @@ pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option, event_sender: Option) -> Result<()> { let check_oom = out_of_memory(min_available_memory_bytes); let common = config.common().clone(); - let machine_id = get_machine_id().await?; + let machine_id = common.machine_identity.machine_id; let task_logger = if let Some(logs) = common.logs.clone() { let rx = onefuzz_telemetry::subscribe_to_events()?; diff --git a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs index aa5e3e80ef..9717cae3e7 100644 --- a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs @@ -198,7 +198,7 @@ pub async fn run_tool( let target_exe = try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .input_path(&input) diff --git a/src/agent/onefuzz-task/src/tasks/config.rs b/src/agent/onefuzz-task/src/tasks/config.rs index a128b2fe84..1a31c39e86 100644 --- a/src/agent/onefuzz-task/src/tasks/config.rs +++ b/src/agent/onefuzz-task/src/tasks/config.rs @@ -10,7 +10,7 @@ use crate::tasks::{ merge, regression, report, }; use anyhow::Result; -use onefuzz::machine_id::{get_machine_id, get_scaleset_name}; +use onefuzz::machine_id::MachineIdentity; use onefuzz_telemetry::{ self as telemetry, Event::task_start, EventData, InstanceTelemetryKey, MicrosoftTelemetryKey, Role, @@ -58,6 +58,8 @@ pub struct CommonConfig { /// Can be disabled by setting to 0. #[serde(default = "default_min_available_memory_mb")] pub min_available_memory_mb: u64, + + pub machine_identity: MachineIdentity, } impl CommonConfig { @@ -67,8 +69,15 @@ impl CommonConfig { ) -> Result> { match &self.heartbeat_queue { Some(url) => { - let hb = init_task_heartbeat(url.clone(), self.task_id, self.job_id, initial_delay) - .await?; + let hb = init_task_heartbeat( + url.clone(), + self.task_id, + self.job_id, + initial_delay, + self.machine_identity.machine_id, + self.machine_identity.machine_name.clone(), + ) + .await?; Ok(Some(hb)) } None => Ok(None), @@ -215,13 +224,14 @@ impl Config { pub async fn run(self) -> Result<()> { telemetry::set_property(EventData::JobId(self.common().job_id)); telemetry::set_property(EventData::TaskId(self.common().task_id)); - telemetry::set_property(EventData::MachineId(get_machine_id().await?)); + telemetry::set_property(EventData::MachineId( + self.common().machine_identity.machine_id, + )); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); telemetry::set_property(EventData::InstanceId(self.common().instance_id)); telemetry::set_property(EventData::Role(Role::Agent)); - let scaleset = get_scaleset_name().await?; - if let Some(scaleset_name) = &scaleset { + if let Some(scaleset_name) = &self.common().machine_identity.scaleset_name { telemetry::set_property(EventData::ScalesetId(scaleset_name.to_string())); } diff --git a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs index 6e0df226aa..d4f3c72c81 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs @@ -266,7 +266,7 @@ impl<'a> TaskContext<'a> { // Try to expand `target_exe` with support for `{tools_dir}`. // // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. - let expand = Expand::new().tools_dir(tools_dir); + let expand = Expand::new(&self.config.common.machine_identity).tools_dir(tools_dir); let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; let expanded_path = Path::new(&expanded); @@ -293,7 +293,7 @@ impl<'a> TaskContext<'a> { async fn command_for_input(&self, input: &Path) -> Result { let target_exe = self.target_exe().await?; - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .input_path(input) diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 2acc4458a3..535ee5660e 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -288,7 +288,7 @@ impl<'a> TaskContext<'a> { try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) .await?; - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .input_path(input) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs index 36e55eb82e..a9dad3e48c 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs @@ -102,6 +102,7 @@ impl GeneratorTask { &target_exe, &self.config.target_options, &self.config.target_env, + self.config.common.machine_identity.clone(), ) .check_asan_log(self.config.check_asan_log) .check_debugger(self.config.check_debugger) @@ -163,7 +164,7 @@ impl GeneratorTask { ) -> Result<()> { utils::reset_tmp_dir(&output_dir).await?; let (mut generator, generator_path) = { - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .machine_id() .await? .setup_dir(&self.config.common.setup_dir) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs index ba08e22842..9bf36eb5a0 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs @@ -86,6 +86,7 @@ impl common::LibFuzzerType for LibFuzzerDotnet { options, env, &config.common.setup_dir, + config.common.machine_identity.clone(), )) } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs index 51da454f0d..1d9d118d9f 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs @@ -28,6 +28,7 @@ impl common::LibFuzzerType for GenericLibFuzzer { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), )) } } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs index 05955afcd7..df71249c09 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs @@ -144,7 +144,7 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { let monitor_path = if let Some(stats_file) = &config.stats_file { Some( - Expand::new() + Expand::new(&config.common.machine_identity) .machine_id() .await? .runtime_dir(runtime_dir.path()) @@ -204,7 +204,7 @@ async fn start_supervisor( None }; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .supervisor_exe(&config.supervisor_exe) diff --git a/src/agent/onefuzz-task/src/tasks/heartbeat.rs b/src/agent/onefuzz-task/src/tasks/heartbeat.rs index a82ee5b649..515fa39d0c 100644 --- a/src/agent/onefuzz-task/src/tasks/heartbeat.rs +++ b/src/agent/onefuzz-task/src/tasks/heartbeat.rs @@ -2,7 +2,6 @@ // Licensed under the MIT License. use crate::onefuzz::heartbeat::HeartbeatClient; -use crate::onefuzz::machine_id::{get_machine_id, get_machine_name}; use anyhow::Result; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -40,9 +39,9 @@ pub async fn init_task_heartbeat( task_id: Uuid, job_id: Uuid, initial_delay: Option, + machine_id: Uuid, + machine_name: String, ) -> Result { - let machine_id = get_machine_id().await?; - let machine_name = get_machine_name().await?; let hb = HeartbeatClient::init_heartbeat( TaskContext { task_id, diff --git a/src/agent/onefuzz-task/src/tasks/merge/generic.rs b/src/agent/onefuzz-task/src/tasks/merge/generic.rs index b1b6265093..de804154cf 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/generic.rs @@ -130,7 +130,7 @@ async fn merge(config: &Config, output_dir: impl AsRef) -> Result<()> { let target_exe = try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?; - let expand = Expand::new() + let expand = Expand::new(&config.common.machine_identity) .machine_id() .await? .input_marker(&config.supervisor_input_marker) diff --git a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs index b119452207..7b8c6f7b43 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs @@ -46,6 +46,7 @@ pub async fn spawn(config: Arc) -> Result<()> { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), ); fuzzer.verify(config.check_fuzzer_help, None).await?; @@ -159,6 +160,7 @@ pub async fn merge_inputs( config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.machine_identity.clone(), ); merger .merge(&config.unique_inputs.local_path, &candidates) diff --git a/src/agent/onefuzz-task/src/tasks/regression/generic.rs b/src/agent/onefuzz-task/src/tasks/regression/generic.rs index cbc1e29965..68d0f1643c 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/generic.rs @@ -74,6 +74,7 @@ impl RegressionHandler for GenericRegressionTask { check_asan_log: self.config.check_asan_log, check_debugger: self.config.check_debugger, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; generic::test_input(args).await } diff --git a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs index ed0322cc04..5a5ce3990b 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs @@ -71,6 +71,7 @@ impl RegressionHandler for LibFuzzerRegressionTask { target_timeout: self.config.target_timeout, check_retry_count: self.config.check_retry_count, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; libfuzzer_report::test_input(args).await } diff --git a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs index a119f303c4..894d1615ee 100644 --- a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs @@ -132,7 +132,7 @@ impl AsanProcessor { // Try to expand `target_exe` with support for `{tools_dir}`. // // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. - let expand = Expand::new().tools_dir(tools_dir); + let expand = Expand::new(&self.config.common.machine_identity).tools_dir(tools_dir); let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; let expanded_path = Path::new(&expanded); @@ -180,7 +180,7 @@ impl AsanProcessor { let mut args = vec![target_exe]; args.extend(self.config.target_options.clone()); - let expand = Expand::new() + let expand = Expand::new(&self.config.common.machine_identity) .input_path(input) .setup_dir(&self.config.common.setup_dir); let expanded_args = expand.evaluate(&args)?; diff --git a/src/agent/onefuzz-task/src/tasks/report/generic.rs b/src/agent/onefuzz-task/src/tasks/report/generic.rs index 490f4e8a8e..2b3c02c955 100644 --- a/src/agent/onefuzz-task/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/generic.rs @@ -10,7 +10,9 @@ use crate::tasks::{ }; use anyhow::{Context, Result}; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, input_tester::Tester, sha256, syncdir::SyncedDir}; +use onefuzz::{ + blob::BlobUrl, input_tester::Tester, machine_id::MachineIdentity, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -118,6 +120,7 @@ pub struct TestInputArgs<'a> { pub check_asan_log: bool, pub check_debugger: bool, pub minimized_stack_depth: Option, + pub machine_identity: MachineIdentity, } pub async fn test_input(args: TestInputArgs<'_>) -> Result { @@ -126,6 +129,7 @@ pub async fn test_input(args: TestInputArgs<'_>) -> Result { args.target_exe, args.target_options, args.target_env, + args.machine_identity.clone(), ) .check_asan_log(args.check_asan_log) .check_debugger(args.check_debugger) @@ -211,6 +215,7 @@ impl<'a> GenericReportProcessor<'a> { check_asan_log: self.config.check_asan_log, check_debugger: self.config.check_debugger, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; test_input(args).await.context("test input failed") } diff --git a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs index 47e51b0a0d..58d9612179 100644 --- a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs @@ -10,7 +10,9 @@ use crate::tasks::{ }; use anyhow::{Context, Result}; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir}; +use onefuzz::{ + blob::BlobUrl, libfuzzer::LibFuzzer, machine_id::MachineIdentity, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -74,6 +76,7 @@ impl ReportTask { self.config.target_options.clone(), self.config.target_env.clone(), &self.config.common.setup_dir, + self.config.common.machine_identity.clone(), ); fuzzer.verify(self.config.check_fuzzer_help, None).await } @@ -120,6 +123,7 @@ pub struct TestInputArgs<'a> { pub target_timeout: Option, pub check_retry_count: u64, pub minimized_stack_depth: Option, + pub machine_identity: MachineIdentity, } pub async fn test_input(args: TestInputArgs<'_>) -> Result { @@ -128,6 +132,7 @@ pub async fn test_input(args: TestInputArgs<'_>) -> Result { args.target_options.to_vec(), args.target_env.clone(), args.setup_dir, + args.machine_identity, ); let task_id = args.task_id; @@ -215,6 +220,7 @@ impl AsanProcessor { target_timeout: self.config.target_timeout, check_retry_count: self.config.check_retry_count, minimized_stack_depth: self.config.minimized_stack_depth, + machine_identity: self.config.common.machine_identity.clone(), }; let result = test_input(args).await?; diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index d0986d615e..0317aa282b 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{machine_id::get_machine_id, sha256::digest_file_blocking}; +use crate::{machine_id::MachineIdentity, sha256::digest_file_blocking}; use anyhow::{format_err, Context, Result}; use onefuzz_telemetry::{InstanceTelemetryKey, MicrosoftTelemetryKey}; use std::path::{Path, PathBuf}; @@ -89,16 +89,11 @@ impl PlaceHolder { pub struct Expand<'a> { values: HashMap>, -} - -impl Default for Expand<'_> { - fn default() -> Self { - Self::new() - } + machine_identity: &'a MachineIdentity, } impl<'a> Expand<'a> { - pub fn new() -> Self { + pub fn new(machine_identity: &'a MachineIdentity) -> Self { let mut values = HashMap::new(); values.insert( PlaceHolder::InputFileNameNoExt.get_string(), @@ -113,12 +108,15 @@ impl<'a> Expand<'a> { ExpandedValue::Mapping(Box::new(Expand::input_file_sha256)), ); - Self { values } + Self { + values, + machine_identity, + } } // Must be manually called to enable the use of async library code. pub async fn machine_id(self) -> Result> { - let id = get_machine_id().await?; + let id = self.machine_identity.machine_id; let value = id.to_string(); Ok(self.set_value(PlaceHolder::MachineId, ExpandedValue::Scalar(value))) } @@ -171,7 +169,10 @@ impl<'a> Expand<'a> { pub fn set_value(self, name: PlaceHolder, value: ExpandedValue<'a>) -> Self { let mut values = self.values; values.insert(name.get_string(), value); - Self { values } + Self { + values, + machine_identity: self.machine_identity, + } } pub fn set_optional_ref<'l, T: 'l>( @@ -412,6 +413,8 @@ impl<'a> Expand<'a> { #[cfg(test)] mod tests { + use crate::machine_id::MachineIdentity; + use super::Expand; use anyhow::{Context, Result}; use std::path::Path; @@ -421,7 +424,7 @@ mod tests { fn test_expand_nested() -> Result<()> { let supervisor_options = vec!["{target_options}".to_string()]; let target_options: Vec<_> = vec!["a", "b", "c"].iter().map(|p| p.to_string()).collect(); - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .target_options(&target_options) .evaluate(&supervisor_options)?; let expected = vec!["a b c"]; @@ -464,7 +467,7 @@ mod tests { let input_corpus_dir = "src"; let generated_inputs_dir = "src"; - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .input_corpus(Path::new(input_corpus_dir)) .generated_inputs(Path::new(generated_inputs_dir)) .target_options(&my_options) @@ -498,14 +501,16 @@ mod tests { ] ); - assert!(Expand::new().evaluate(&my_args).is_err()); + assert!(Expand::new(&MachineIdentity::default()) + .evaluate(&my_args) + .is_err()); Ok(()) } #[test] fn test_expand_in_string() -> Result<()> { - let result = Expand::new() + let result = Expand::new(&MachineIdentity::default()) .input_path("src/lib.rs") .evaluate_value("a {input} b")?; assert!(result.contains("lib.rs")); @@ -514,10 +519,17 @@ mod tests { #[tokio::test] async fn test_expand_machine_id() -> Result<()> { - let expand = Expand::new().machine_id().await?; + let machine_id = Uuid::new_v4(); + let expand = Expand::new(&MachineIdentity { + machine_id: machine_id, + ..Default::default() + }) + .machine_id() + .await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. - Uuid::parse_str(&expanded)?; + let expanded_machine_id = Uuid::parse_str(&expanded)?; + assert_eq!(expanded_machine_id, machine_id); Ok(()) } } diff --git a/src/agent/onefuzz/src/input_tester.rs b/src/agent/onefuzz/src/input_tester.rs index 6eb9e0a655..ec4375455d 100644 --- a/src/agent/onefuzz/src/input_tester.rs +++ b/src/agent/onefuzz/src/input_tester.rs @@ -7,6 +7,7 @@ use crate::{ asan::{add_asan_log_env, check_asan_path, check_asan_string}, env::{get_path_with_directory, update_path, LD_LIBRARY_PATH, PATH}, expand::Expand, + machine_id::MachineIdentity, process::run_cmd, }; use anyhow::{Context, Error, Result}; @@ -36,6 +37,7 @@ pub struct Tester<'a> { check_retry_count: u64, add_setup_to_ld_library_path: bool, add_setup_to_path: bool, + machine_identity: MachineIdentity, } #[derive(Debug)] @@ -57,6 +59,7 @@ impl<'a> Tester<'a> { exe_path: &'a Path, arguments: &'a [String], environ: &'a HashMap, + machine_identity: MachineIdentity, ) -> Self { Self { setup_dir, @@ -70,6 +73,7 @@ impl<'a> Tester<'a> { check_retry_count: 0, add_setup_to_ld_library_path: false, add_setup_to_path: false, + machine_identity, } } @@ -289,7 +293,7 @@ impl<'a> Tester<'a> { }; let (argv, env) = { - let expand = Expand::new() + let expand = Expand::new(&self.machine_identity) .machine_id() .await? .input_path(input_file) diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index d221732639..43eb4a4fae 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -6,6 +6,7 @@ use crate::{ expand::Expand, fs::{list_files, write_file}, input_tester::{TestResult, Tester}, + machine_id::MachineIdentity, }; use anyhow::{Context, Result}; use rand::seq::SliceRandom; @@ -39,6 +40,7 @@ pub struct LibFuzzer { exe: PathBuf, options: Vec, env: HashMap, + machine_identity: MachineIdentity, } impl LibFuzzer { @@ -47,12 +49,14 @@ impl LibFuzzer { options: Vec, env: HashMap, setup_dir: impl Into, + machine_identity: MachineIdentity, ) -> Self { Self { exe: exe.into(), options, env, setup_dir: setup_dir.into(), + machine_identity, } } @@ -98,7 +102,7 @@ impl LibFuzzer { ); } - let expand = Expand::new() + let expand = Expand::new(&self.machine_identity) .machine_id() .await? .target_exe(&self.exe) @@ -310,11 +314,17 @@ impl LibFuzzer { let mut options = self.options.clone(); options.push("{input}".to_string()); - let mut tester = Tester::new(&self.setup_dir, &self.exe, &options, &self.env) - .check_asan_stderr(true) - .check_retry_count(retry) - .add_setup_to_path(true) - .set_optional(timeout, |tester, timeout| tester.timeout(timeout)); + let mut tester = Tester::new( + &self.setup_dir, + &self.exe, + &options, + &self.env, + self.machine_identity.clone(), + ) + .check_asan_stderr(true) + .check_retry_count(retry) + .add_setup_to_path(true) + .set_optional(timeout, |tester, timeout| tester.timeout(timeout)); if cfg!(target_family = "unix") { tester = tester.add_setup_to_ld_library_path(true); diff --git a/src/agent/onefuzz/src/machine_id.rs b/src/agent/onefuzz/src/machine_id.rs index 33b222776d..d608a95e3d 100644 --- a/src/agent/onefuzz/src/machine_id.rs +++ b/src/agent/onefuzz/src/machine_id.rs @@ -10,6 +10,13 @@ use std::time::Duration; use tokio::fs; use uuid::Uuid; +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Default, Serialize)] +pub struct MachineIdentity { + pub machine_id: Uuid, + pub machine_name: String, + pub scaleset_name: Option, +} + // https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service#tracking-vm-running-on-azure const IMS_ID_URL: &str = "http://169.254.169.254/metadata/instance/compute/vmId?api-version=2020-06-01&format=text"; @@ -19,127 +26,156 @@ const VM_NAME_URL: &str = "http://169.254.169.254/metadata/instance/compute/name?api-version=2020-06-01&format=text"; const VM_SCALESET_NAME: &str = - "http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2020-06-01&format=text"; - -pub async fn get_ims_id() -> Result { - let path = onefuzz_etc()?.join("ims_id"); - let body = match fs::read_to_string(&path).await { - Ok(body) => body, - Err(_) => { - let resp = reqwest::Client::new() - .get(IMS_ID_URL) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - .context("get_ims_id")?; - let body = resp.text().await?; - write_file(path, &body).await?; - body - } - }; - - let value = Uuid::parse_str(&body)?; - Ok(value) -} +"http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2020-06-01&format=text"; + +const COMPUTE_METADATA_URL: &str = + "http://169.254.169.254/metadata/instance/compute?api-version=2020-06-01"; + +impl MachineIdentity { + pub async fn from_metadata() -> Result { + let machine_id = Self::get_machine_id().await?; + let machine_name = Self::get_machine_name().await?; + let scaleset_name = Self::get_scaleset_name().await?; + + Ok(Self { + machine_id, + machine_name, + scaleset_name, + }) + } -pub async fn get_machine_name() -> Result { - let path = onefuzz_etc()?.join("machine_name"); - let body = match fs::read_to_string(&path).await { - Ok(body) => body, - Err(_) => { - let resp = reqwest::Client::new() - .get(VM_NAME_URL) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - .context("get_machine_name")?; - let body = resp.text().await?; - write_file(path, &body).await?; - body - } - }; + pub fn from_env() -> Result { + let machine_id = Uuid::parse_str(&std::env::var("ONEFUZZ_MACHINE_ID")?)?; + let machine_name = std::env::var("ONEFUZZ_MACHINE_NAME")?; + let scaleset_name = std::env::var("ONEFUZZ_SCALESET_NAME").ok(); - Ok(body) -} + Ok(Self { + machine_id, + machine_name, + scaleset_name, + }) + } -pub async fn get_scaleset_name() -> Result> { - let path = onefuzz_etc()?.join("scaleset_name"); - if let Ok(scaleset_name) = fs::read_to_string(&path).await { - return Ok(Some(scaleset_name)); + pub async fn get_ims_id() -> Result { + let path = onefuzz_etc()?.join("ims_id"); + let body = match fs::read_to_string(&path).await { + Ok(body) => body, + Err(_) => { + let resp = reqwest::Client::new() + .get(IMS_ID_URL) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + .context("get_ims_id")?; + let body = resp.text().await?; + write_file(path, &body).await?; + body + } + }; + + let value = Uuid::parse_str(&body)?; + Ok(value) } - if let Ok(resp) = reqwest::Client::new() - .get(VM_SCALESET_NAME) - .timeout(Duration::from_millis(500)) - .header("Metadata", "true") - .send_retry_default() - .await - { - let body = resp.text().await?; - write_file(path, &body).await?; - Ok(Some(body)) - } else { - Ok(None) + pub async fn get_machine_name() -> Result { + let path = onefuzz_etc()?.join("machine_name"); + let body = match fs::read_to_string(&path).await { + Ok(body) => body, + Err(_) => { + let resp = reqwest::Client::new() + .get(VM_NAME_URL) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + .context("get_machine_name")?; + let body = resp.text().await?; + write_file(path, &body).await?; + body + } + }; + + Ok(body) } -} -#[cfg(target_os = "linux")] -pub async fn get_os_machine_id() -> Result { - let path = Path::new("/etc/machine-id"); - let contents = fs::read_to_string(&path) - .await - .with_context(|| format!("unable to read machine_id: {}", path.display()))?; - let uuid = Uuid::parse_str(contents.trim())?; - Ok(uuid) -} + pub async fn get_scaleset_name() -> Result> { + let path = onefuzz_etc()?.join("scaleset_name"); + if let Ok(scaleset_name) = fs::read_to_string(&path).await { + return Ok(Some(scaleset_name)); + } -#[cfg(target_os = "windows")] -pub async fn get_os_machine_id() -> Result { - use winreg::enums::{HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_64KEY}; - use winreg::RegKey; - - let key: &str = "SOFTWARE\\Microsoft\\Cryptography"; - - let hklm = RegKey::predef(HKEY_LOCAL_MACHINE); - let crypt = if let Ok(crypt) = hklm.open_subkey_with_flags(key, KEY_READ) { - crypt - } else { - hklm.open_subkey_with_flags(key, KEY_READ | KEY_WOW64_64KEY)? - }; - let guid: String = crypt.get_value("MachineGuid")?; - Ok(Uuid::parse_str(&guid)?) -} + if let Ok(resp) = reqwest::Client::new() + .get(VM_SCALESET_NAME) + .timeout(Duration::from_millis(500)) + .header("Metadata", "true") + .send_retry_default() + .await + { + let body = resp.text().await?; + write_file(path, &body).await?; + Ok(Some(body)) + } else { + Ok(None) + } + } -async fn get_machine_id_impl() -> Result { - let ims_id = get_ims_id().await; - if ims_id.is_ok() { - return ims_id; + #[cfg(target_os = "linux")] + pub async fn get_os_machine_id() -> Result { + let path = Path::new("/etc/machine-id"); + let contents = fs::read_to_string(&path) + .await + .with_context(|| format!("unable to read machine_id: {}", path.display()))?; + let uuid = Uuid::parse_str(contents.trim())?; + Ok(uuid) } - let machine_id = get_os_machine_id().await; - if machine_id.is_ok() { - return machine_id; + #[cfg(target_os = "windows")] + pub async fn get_os_machine_id() -> Result { + use winreg::enums::{HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_64KEY}; + use winreg::RegKey; + + let key: &str = "SOFTWARE\\Microsoft\\Cryptography"; + + let hklm = RegKey::predef(HKEY_LOCAL_MACHINE); + let crypt = if let Ok(crypt) = hklm.open_subkey_with_flags(key, KEY_READ) { + crypt + } else { + hklm.open_subkey_with_flags(key, KEY_READ | KEY_WOW64_64KEY)? + }; + let guid: String = crypt.get_value("MachineGuid")?; + Ok(Uuid::parse_str(&guid)?) } - Ok(Uuid::new_v4()) -} + async fn get_machine_id_impl() -> Result { + let ims_id = Self::get_ims_id().await; + if ims_id.is_ok() { + return ims_id; + } -pub async fn get_machine_id() -> Result { - let path = onefuzz_etc()?.join("machine_id"); - let result = match fs::read_to_string(&path).await { - Ok(body) => Uuid::parse_str(&body)?, - Err(_) => { - let value = get_machine_id_impl().await?; - write_file(path, &value.to_string()).await?; - value + let machine_id = Self::get_os_machine_id().await; + if machine_id.is_ok() { + return machine_id; } - }; - Ok(result) + + Ok(Uuid::new_v4()) + } + + pub async fn get_machine_id() -> Result { + let path = onefuzz_etc()?.join("machine_id"); + let result = match fs::read_to_string(&path).await { + Ok(body) => Uuid::parse_str(&body)?, + Err(_) => { + let value = Self::get_machine_id_impl().await?; + write_file(path, &value.to_string()).await?; + value + } + }; + Ok(result) + } } #[tokio::test] async fn test_get_machine_id() { - get_os_machine_id().await.unwrap(); + MachineIdentity::get_os_machine_id().await.unwrap(); } From 9d1dbb3ac08bc9e96996fec20e71dd6781b6e2d2 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 22 Nov 2022 12:33:15 -0800 Subject: [PATCH 34/52] build fix --- src/agent/onefuzz-agent/src/commands.rs | 5 ----- src/deny.toml | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/agent/onefuzz-agent/src/commands.rs b/src/agent/onefuzz-agent/src/commands.rs index 92e3fe3791..081167fb5d 100644 --- a/src/agent/onefuzz-agent/src/commands.rs +++ b/src/agent/onefuzz-agent/src/commands.rs @@ -155,11 +155,6 @@ pub async fn add_ssh_key(key_info: &SshKeyInfo) -> Result<()> { #[cfg(target_family = "unix")] pub async fn add_ssh_key(key_info: &SshKeyInfo) -> Result<()> { - if get_scaleset_name().await?.is_none() { - warn!("adding ssh keys only supported on managed nodes"); - return Ok(()); - } - let user = get_user_by_name(ONEFUZZ_SERVICE_USER).ok_or_else(|| format_err!("unable to find user"))?; info!("adding ssh key:{:?} to user:{:?}", key_info, user); diff --git a/src/deny.toml b/src/deny.toml index 8fa38dc088..98c5446d0a 100644 --- a/src/deny.toml +++ b/src/deny.toml @@ -17,6 +17,7 @@ yanked = "deny" ignore = [ "RUSTSEC-2022-0048", # xml-rs is unmaintained "RUSTSEC-2021-0139", # ansi_term is unmaintained + "RUSTSEC-2021-0145", # atty bug: we are unaffected (no custom allocator) ] [bans] From be6092eebe8fae68437fee2b94a91044d76df281 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 22 Nov 2022 12:54:27 -0800 Subject: [PATCH 35/52] build fix --- src/agent/onefuzz-agent/src/agent.rs | 1 + src/agent/onefuzz-agent/src/agent/tests.rs | 1 + src/agent/onefuzz/examples/test-input.rs | 4 ++-- src/agent/onefuzz/src/expand.rs | 9 ++++----- src/agent/onefuzz/src/libfuzzer.rs | 2 ++ src/agent/onefuzz/src/machine_id.rs | 3 --- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index 1531471fe3..f0cdb1b244 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#![allow(clippy::too_many_arguments)] use anyhow::{Error, Result}; use tokio::time; diff --git a/src/agent/onefuzz-agent/src/agent/tests.rs b/src/agent/onefuzz-agent/src/agent/tests.rs index 23d263ad00..a27c7365fc 100644 --- a/src/agent/onefuzz-agent/src/agent/tests.rs +++ b/src/agent/onefuzz-agent/src/agent/tests.rs @@ -35,6 +35,7 @@ impl Fixture { work_queue, worker_runner, None, + true, ) } diff --git a/src/agent/onefuzz/examples/test-input.rs b/src/agent/onefuzz/examples/test-input.rs index 7066090087..adf18e7086 100644 --- a/src/agent/onefuzz/examples/test-input.rs +++ b/src/agent/onefuzz/examples/test-input.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use anyhow::Result; -use onefuzz::input_tester::Tester; +use onefuzz::{input_tester::Tester, machine_id::MachineIdentity}; use structopt::StructOpt; #[derive(Debug, PartialEq, Eq, StructOpt)] @@ -53,7 +53,7 @@ async fn main() -> Result<()> { } let env = Default::default(); - let tester = Tester::new(&setup_dir, &opt.exe, &target_options, &env); + let tester = Tester::new(&setup_dir, &opt.exe, &target_options, &env, MachineIdentity::default()); let check_debugger = !opt.no_check_debugger; let tester = tester diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index 0317aa282b..73159f80c0 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -520,12 +520,11 @@ mod tests { #[tokio::test] async fn test_expand_machine_id() -> Result<()> { let machine_id = Uuid::new_v4(); - let expand = Expand::new(&MachineIdentity { - machine_id: machine_id, + let machine_identity = MachineIdentity { + machine_id, ..Default::default() - }) - .machine_id() - .await?; + }; + let expand = Expand::new(&machine_identity).machine_id().await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. let expanded_machine_id = Uuid::parse_str(&expanded)?; diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 43eb4a4fae..ec6aeec262 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -445,6 +445,7 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), + MachineIdentity::default(), ); // verify catching bad exits with -help=1 @@ -473,6 +474,7 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), + MachineIdentity::default(), ); // verify good exits with -help=1 assert!( diff --git a/src/agent/onefuzz/src/machine_id.rs b/src/agent/onefuzz/src/machine_id.rs index d608a95e3d..f0949b28ad 100644 --- a/src/agent/onefuzz/src/machine_id.rs +++ b/src/agent/onefuzz/src/machine_id.rs @@ -28,9 +28,6 @@ const VM_NAME_URL: &str = const VM_SCALESET_NAME: &str = "http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2020-06-01&format=text"; -const COMPUTE_METADATA_URL: &str = - "http://169.254.169.254/metadata/instance/compute?api-version=2020-06-01"; - impl MachineIdentity { pub async fn from_metadata() -> Result { let machine_id = Self::get_machine_id().await?; From f36dd9d89dc514c3920dad6c7a001c5c09ae19f1 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 22 Nov 2022 13:55:13 -0800 Subject: [PATCH 36/52] format --- src/agent/onefuzz/examples/test-input.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/agent/onefuzz/examples/test-input.rs b/src/agent/onefuzz/examples/test-input.rs index adf18e7086..3f83df9107 100644 --- a/src/agent/onefuzz/examples/test-input.rs +++ b/src/agent/onefuzz/examples/test-input.rs @@ -53,7 +53,13 @@ async fn main() -> Result<()> { } let env = Default::default(); - let tester = Tester::new(&setup_dir, &opt.exe, &target_options, &env, MachineIdentity::default()); + let tester = Tester::new( + &setup_dir, + &opt.exe, + &target_options, + &env, + MachineIdentity::default(), + ); let check_debugger = !opt.no_check_debugger; let tester = tester From 3df523f271185c3a6cf20c4ff5409c7c18480387 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 23 Nov 2022 09:05:13 -0800 Subject: [PATCH 37/52] address pr comment --- src/ApiService/ApiService/Functions/AgentCanSchedule.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs index 63d1d71fe4..af41c3efe7 100644 --- a/src/ApiService/ApiService/Functions/AgentCanSchedule.cs +++ b/src/ApiService/ApiService/Functions/AgentCanSchedule.cs @@ -56,8 +56,7 @@ private async Async.Task Post(HttpRequestData req) { if (workStopped) { _log.Info($"Work stopped for: {canScheduleRequest.MachineId:Tag:MachineId} and {canScheduleRequest.TaskId:Tag:TaskId}"); - allowed = false; - reason = "Work stopped"; + return await RequestHandling.Ok(req, new CanSchedule(Allowed: false, WorkStopped: workStopped, Reason: "Work stopped")); } var scp = await _context.NodeOperations.AcquireScaleInProtection(node); From 8a127f66913357ccbbb8587aa177d3680959f137 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 23 Nov 2022 11:27:25 -0800 Subject: [PATCH 38/52] build fix --- src/deny.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/deny.toml b/src/deny.toml index 8fa38dc088..bc8c577e24 100644 --- a/src/deny.toml +++ b/src/deny.toml @@ -17,6 +17,8 @@ yanked = "deny" ignore = [ "RUSTSEC-2022-0048", # xml-rs is unmaintained "RUSTSEC-2021-0139", # ansi_term is unmaintained + "RUSTSEC-2021-0145", # atty bug: we are unaffected (no custom allocator) +] ] [bans] From c7293366eff1b67fbe88d47ba763f45cbd820068 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 23 Nov 2022 15:35:32 -0800 Subject: [PATCH 39/52] get baseAddress from environment variable --- src/ApiService/ApiService/Functions/AgentRegistration.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index c23fa0b00f..c35871a5d2 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -70,9 +70,11 @@ private async Async.Task Get(HttpRequestData req) { } private async Async.Task CreateRegistrationResponse(Service.Pool pool) { - var baseAddress = _context.Creds.GetInstanceUrl(); - var eventsUrl = new Uri(baseAddress, "/api/agents/events"); - var commandsUrl = new Uri(baseAddress, "/api/agents/commands"); + var hostName = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME"); + var scheme = Environment.GetEnvironmentVariable("HTTPS") != null? "https" : "http"; + var baseAddress = $"{scheme}://{hostName}"; + var eventsUrl = new Uri($"{baseAddress}/api/agents/events"); + var commandsUrl = new Uri($"{baseAddress}/api/agents/commands"); var workQueue = await _context.Queue.GetQueueSas( _context.PoolOperations.GetPoolQueue(pool.PoolId), From 1d4add53540bca19950b8fbcf91cf2527724ce36 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Mon, 28 Nov 2022 09:17:06 -0800 Subject: [PATCH 40/52] adding machine_id and machine_name paramters --- src/agent/onefuzz-agent/src/config.rs | 23 +++++++++++++++++++---- src/agent/onefuzz-agent/src/main.rs | 18 ++++++++++++++++++ src/agent/onefuzz/src/auth.rs | 13 +++++++------ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 5d5f65a9ea..89e3316883 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -3,7 +3,7 @@ use anyhow::{Context, Result}; use onefuzz::{ - auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}, + auth::{ClientCredentials, Credentials, ManagedIdentityCredentials, Secret}, http::{is_auth_error_code, ResponseExt}, jitter::delay_with_jitter, machine_id::MachineIdentity, @@ -46,10 +46,18 @@ fn default_as_true() -> bool { true } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct RawClientCredentials { + client_id: Uuid, + client_secret: Secret, + tenant: String, + multi_tenant_domain: Option, +} + // Temporary shim type to bridge the current service-provided config. #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct RawStaticConfig { - pub client_credentials: Option, + pub client_credentials: Option, pub pool_name: String, @@ -76,7 +84,14 @@ impl StaticConfig { let config: RawStaticConfig = serde_json::from_slice(data)?; let credentials = match config.client_credentials { - Some(client) => client.into(), + Some(client) => ClientCredentials::new( + client.client_id, + client.client_secret, + config.onefuzz_url.to_string(), + client.tenant, + client.multi_tenant_domain, + ) + .into(), None => { // Remove trailing `/`, which is treated as a distinct resource. let resource = config @@ -150,7 +165,7 @@ impl StaticConfig { let credentials = ClientCredentials::new( client_id, - client_secret, + client_secret.into(), onefuzz_url.to_string(), tenant, multi_tenant_domain.clone(), diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index ddb549d072..6edd2efcc8 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -21,6 +21,7 @@ use std::process::{Command, Stdio}; use anyhow::{Context, Result}; use clap::Parser; +use onefuzz::machine_id::MachineIdentity; use onefuzz::process::ExitStatus; use onefuzz_telemetry::{self as telemetry, EventData, Role}; use std::io::{self, Write}; @@ -59,6 +60,12 @@ struct RunOpt { /// the specified directory #[clap(short, long = "--redirect-output", parse(from_os_str))] redirect_output: Option, + + #[clap(long = "--machine_id")] + machine_id: Option, + + #[clap(long = "--machine_name")] + machine_name: Option, } fn main() -> Result<()> { @@ -171,6 +178,8 @@ fn run(opt: RunOpt) -> Result<()> { // We can't send telemetry if this fails. let rt = tokio::runtime::Runtime::new()?; + let opt_machine_id = opt.machine_id.clone(); + let opt_machine_name = opt.machine_name.clone(); let config = rt.block_on(load_config(opt)); // We can't send telemetry, because we couldn't get a telemetry key from the config. @@ -181,6 +190,15 @@ fn run(opt: RunOpt) -> Result<()> { let config = config?; + let config = StaticConfig { + machine_identity: MachineIdentity { + machine_id: opt_machine_id.unwrap_or(config.machine_identity.machine_id), + machine_name: opt_machine_name.unwrap_or(config.machine_identity.machine_name), + ..config.machine_identity + }, + ..config + }; + let result = rt.block_on(run_agent(config)); if let Err(err) = &result { diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs index d25a3807f9..c54317af2b 100644 --- a/src/agent/onefuzz/src/auth.rs +++ b/src/agent/onefuzz/src/auth.rs @@ -94,13 +94,11 @@ pub struct ClientCredentials { impl ClientCredentials { pub fn new( client_id: Uuid, - client_secret: String, + client_secret: Secret, resource: String, tenant: String, multi_tenant_domain: Option, ) -> Self { - let client_secret = client_secret.into(); - Self { client_id, client_secret, @@ -113,9 +111,12 @@ impl ClientCredentials { pub async fn access_token(&self) -> Result { let (authority, scope) = { let url = Url::parse(&self.resource.clone())?; - let host = url.host_str().ok_or_else(|| { - anyhow::format_err!("resource URL does not have a host string: {}", url) - })?; + let host = match (url.host_str(), url.port()) { + (Some(host), Some(port)) => format!("{}:{}", host, port), + (Some(host), None) => host.to_string(), + (None, _) => anyhow::bail!("resource URL does not have a host string: {}", url), + }; + if let Some(domain) = &self.multi_tenant_domain { let instance: Vec<&str> = host.split('.').collect(); ( From 0019181d1413c23437bb2c5b6bc2f89afc09ab17 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 29 Nov 2022 08:49:55 -0800 Subject: [PATCH 41/52] more tweaking --- src/agent/onefuzz-agent/src/agent.rs | 5 ++- src/agent/onefuzz-agent/src/config.rs | 4 ++ src/agent/onefuzz-agent/src/coordinator.rs | 2 +- src/agent/onefuzz-agent/src/debug.rs | 6 ++- src/agent/onefuzz-agent/src/done.rs | 23 +++++++--- src/agent/onefuzz-agent/src/main.rs | 45 ++++++++++++------- src/agent/onefuzz-agent/src/scheduler.rs | 10 ++++- src/agent/onefuzz-agent/src/setup.rs | 22 +++++---- src/agent/onefuzz-agent/src/work.rs | 25 ++++++++--- src/agent/onefuzz-agent/src/worker.rs | 10 ++++- .../src/tasks/fuzz/libfuzzer/common.rs | 5 ++- src/agent/onefuzz/src/sanitizer.rs | 5 ++- 12 files changed, 116 insertions(+), 46 deletions(-) diff --git a/src/agent/onefuzz-agent/src/agent.rs b/src/agent/onefuzz-agent/src/agent.rs index 8f4b556b64..d26ecb732a 100644 --- a/src/agent/onefuzz-agent/src/agent.rs +++ b/src/agent/onefuzz-agent/src/agent.rs @@ -28,6 +28,7 @@ pub struct Agent { previous_state: NodeState, last_poll_command: Result, PollCommandError>, managed: bool, + machine_id: uuid::Uuid, } impl Agent { @@ -40,6 +41,7 @@ impl Agent { worker_runner: Box, heartbeat: Option, managed: bool, + machine_id: uuid::Uuid, ) -> Self { let scheduler = Some(scheduler); let previous_state = NodeState::Init; @@ -56,6 +58,7 @@ impl Agent { previous_state, last_poll_command, managed, + machine_id, } } @@ -266,7 +269,7 @@ impl Agent { async fn done(&mut self, state: State) -> Result { debug!("agent done"); - set_done_lock().await?; + set_done_lock(self.machine_id).await?; let event = match state.cause() { DoneCause::SetupError { diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 89e3316883..c02113a84b 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -208,6 +208,10 @@ pub struct DynamicConfig { impl DynamicConfig { pub async fn save(&self) -> Result<()> { let path = Self::save_path()?; + let dir = path + .parent() + .ok_or(anyhow!("invalid dynamic config path"))?; + fs::create_dir_all(dir).await?; let data = serde_json::to_vec(&self)?; fs::write(&path, &data) .await diff --git a/src/agent/onefuzz-agent/src/coordinator.rs b/src/agent/onefuzz-agent/src/coordinator.rs index fb466a1808..2c79e5b1ad 100644 --- a/src/agent/onefuzz-agent/src/coordinator.rs +++ b/src/agent/onefuzz-agent/src/coordinator.rs @@ -183,7 +183,7 @@ pub enum PollCommandError { pub struct Coordinator { client: Client, - registration: Registration, + pub registration: Registration, token: AccessToken, } diff --git a/src/agent/onefuzz-agent/src/debug.rs b/src/agent/onefuzz-agent/src/debug.rs index 692a0bd02b..8dae32bcd9 100644 --- a/src/agent/onefuzz-agent/src/debug.rs +++ b/src/agent/onefuzz-agent/src/debug.rs @@ -179,8 +179,10 @@ fn debug_run_worker(opt: RunWorkerOpt) -> Result<()> { async fn run_worker(mut work_set: WorkSet) -> Result> { use crate::setup::SetupRunner; - - SetupRunner.run(&work_set).await?; + let mut setup_runner = SetupRunner { + machine_id: Uuid::new_v4(), + }; + setup_runner.run(&work_set).await?; let mut events = vec![]; let work_unit = work_set.work_units.pop().unwrap(); diff --git a/src/agent/onefuzz-agent/src/done.rs b/src/agent/onefuzz-agent/src/done.rs index 8c10fc9882..f13933fa5c 100644 --- a/src/agent/onefuzz-agent/src/done.rs +++ b/src/agent/onefuzz-agent/src/done.rs @@ -7,19 +7,30 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use onefuzz::fs::onefuzz_root; use tokio::fs; +use uuid::Uuid; -pub async fn set_done_lock() -> Result<()> { - let path = done_path()?; +pub async fn set_done_lock(machine_id: Uuid) -> Result<()> { + let path = done_path(machine_id)?; fs::write(&path, "") .await .with_context(|| format!("unable to write done lock: {}", path.display()))?; Ok(()) } -pub fn is_agent_done() -> Result { - Ok(metadata(done_path()?).is_ok()) +pub fn remove_done_lock(machine_id: Uuid) -> Result<()> { + let path = done_path(machine_id)?; + if path.exists() { + std::fs::remove_file(&path) + .with_context(|| format!("unable to remove done lock: {}", path.display()))?; + } + + Ok(()) +} + +pub fn is_agent_done(machine_id: Uuid) -> Result { + Ok(metadata(done_path(machine_id)?).is_ok()) } -pub fn done_path() -> Result { - Ok(onefuzz_root()?.join("supervisor-is-done")) +pub fn done_path(machine_id: Uuid) -> Result { + Ok(onefuzz_root()?.join(format!("supervisor-is-done-{}", machine_id))) } diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 6edd2efcc8..e17edbfdc9 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -66,6 +66,9 @@ struct RunOpt { #[clap(long = "--machine_name")] machine_name: Option, + + #[clap(long = "--reset_lock", default_value = "false", takes_value = false)] + reset_node_lock: bool, } fn main() -> Result<()> { @@ -167,19 +170,10 @@ fn run(opt: RunOpt) -> Result<()> { if opt.redirect_output.is_some() { return redirect(opt); } - - if done::is_agent_done()? { - debug!( - "agent is done, remove lock ({}) to continue", - done::done_path()?.display() - ); - return Ok(()); - } - - // We can't send telemetry if this fails. - let rt = tokio::runtime::Runtime::new()?; let opt_machine_id = opt.machine_id.clone(); let opt_machine_name = opt.machine_name.clone(); + let rt = tokio::runtime::Runtime::new()?; + let reset_lock = opt.reset_node_lock; let config = rt.block_on(load_config(opt)); // We can't send telemetry, because we couldn't get a telemetry key from the config. @@ -199,7 +193,17 @@ fn run(opt: RunOpt) -> Result<()> { ..config }; - let result = rt.block_on(run_agent(config)); + if reset_lock { + done::remove_done_lock(config.machine_identity.machine_id)?; + } else if done::is_agent_done(config.machine_identity.machine_id)? { + debug!( + "agent is done, remove lock ({}) to continue", + done::done_path(config.machine_identity.machine_id)?.display() + ); + return Ok(()); + } + + let result = rt.block_on(run_agent(config, reset_lock)); if let Err(err) = &result { error!("error running supervisor agent: {:?}", err); @@ -231,7 +235,7 @@ async fn check_existing_worksets(coordinator: &mut coordinator::Coordinator) -> // that is the case, mark each of the work units within the workset as // failed, then exit as a failure. - if let Some(work) = WorkSet::load_from_fs_context().await? { + if let Some(work) = WorkSet::load_from_fs_context(coordinator.registration.machine_id).await? { warn!("onefuzz-agent unexpectedly identified an existing workset on start"); let failure = match failure::read_failure() { Ok(value) => format!("onefuzz-agent failed: {}", value), @@ -269,17 +273,17 @@ async fn check_existing_worksets(coordinator: &mut coordinator::Coordinator) -> // force set done semaphore, as to not prevent the supervisor continuing // to report the workset as failed. - done::set_done_lock().await?; + done::set_done_lock(coordinator.registration.machine_id).await?; anyhow::bail!( "failed to start due to pre-existing workset config: {}", - WorkSet::context_path()?.display() + WorkSet::context_path(coordinator.registration.machine_id)?.display() ); } Ok(()) } -async fn run_agent(config: StaticConfig) -> Result<()> { +async fn run_agent(config: StaticConfig, reset_node: bool) -> Result<()> { telemetry::set_property(EventData::InstanceId(config.instance_id)); telemetry::set_property(EventData::MachineId(config.machine_identity.machine_id)); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); @@ -306,6 +310,10 @@ async fn run_agent(config: StaticConfig) -> Result<()> { let mut reboot = reboot::Reboot; let reboot_context = reboot.load_context().await?; + if reset_node { + WorkSet::remove_context(config.machine_identity.machine_id).await?; + } + if reboot_context.is_none() { check_existing_worksets(&mut coordinator).await?; } @@ -329,11 +337,14 @@ async fn run_agent(config: StaticConfig) -> Result<()> { Box::new(coordinator), Box::new(reboot), scheduler, - Box::new(setup::SetupRunner), + Box::new(setup::SetupRunner { + machine_id: config.machine_identity.machine_id, + }), Box::new(work_queue), Box::new(worker::WorkerRunner), agent_heartbeat, config.managed, + config.machine_identity.machine_id, ); info!("running agent"); diff --git a/src/agent/onefuzz-agent/src/scheduler.rs b/src/agent/onefuzz-agent/src/scheduler.rs index e9b5fcc8ae..e786d9fa82 100644 --- a/src/agent/onefuzz-agent/src/scheduler.rs +++ b/src/agent/onefuzz-agent/src/scheduler.rs @@ -13,6 +13,7 @@ use crate::setup::ISetupRunner; use crate::work::*; use crate::worker::*; +#[derive(Debug)] pub enum Scheduler { Free(State), SettingUp(State), @@ -97,24 +98,30 @@ impl Default for Scheduler { } } +#[derive(Debug)] pub struct Free; +#[derive(Debug)] pub struct SettingUp { work_set: WorkSet, } +#[derive(Debug)] pub struct PendingReboot { work_set: WorkSet, } +#[derive(Debug)] pub struct Ready { work_set: WorkSet, } +#[derive(Debug)] pub struct Busy { workers: Vec>, } +#[derive(Debug)] pub struct Done { cause: DoneCause, } @@ -138,6 +145,7 @@ impl Context for Ready {} impl Context for Busy {} impl Context for Done {} +#[derive(Debug)] pub struct State { ctx: C, } @@ -201,7 +209,7 @@ impl State { // No script was executed. } Err(err) => { - let error = err.to_string(); + let error = format!("{:?}", err.to_string()); warn!("{}", error); let cause = DoneCause::SetupError { error, diff --git a/src/agent/onefuzz-agent/src/setup.rs b/src/agent/onefuzz-agent/src/setup.rs index 95af22f10e..236e1c722e 100644 --- a/src/agent/onefuzz-agent/src/setup.rs +++ b/src/agent/onefuzz-agent/src/setup.rs @@ -11,6 +11,7 @@ use onefuzz::az_copy; use onefuzz::process::Output; use tokio::fs; use tokio::process::Command; +use uuid::Uuid; use crate::work::*; @@ -36,24 +37,22 @@ impl ISetupRunner for SetupRunner { } #[derive(Clone, Copy, Debug)] -pub struct SetupRunner; +pub struct SetupRunner { + pub machine_id: Uuid, +} impl SetupRunner { pub async fn run(&mut self, work_set: &WorkSet) -> Result { info!("running setup for work set"); - - work_set.save_context().await?; - + work_set.save_context(self.machine_id).await?; // Download the setup container. let setup_url = work_set.setup_url.url()?; let setup_dir = work_set.setup_dir()?; - // `azcopy sync` requires the local dir to exist. fs::create_dir_all(&setup_dir).await.with_context(|| { format!("unable to create setup container: {}", setup_dir.display()) })?; az_copy::sync(setup_url.to_string(), &setup_dir, false).await?; - debug!( "synced setup container from {} to {}", setup_url, @@ -110,12 +109,19 @@ async fn create_setup_symlink(setup_dir: &Path, work_unit: &WorkUnit) -> Result< let working_dir = work_unit.working_dir()?; - fs::create_dir(&working_dir).await.with_context(|| { + let create_work_dir = fs::create_dir(&working_dir).await.with_context(|| { format!( "unable to create working directory: {}", working_dir.display() ) - })?; + }); + + if let Err(err) = create_work_dir { + if !working_dir.exists() { + return Err(err); + } + } + let task_setup_dir = working_dir.join("setup"); // Tokio does not ship async versions of the `std::fs::os` symlink diff --git a/src/agent/onefuzz-agent/src/work.rs b/src/agent/onefuzz-agent/src/work.rs index 2599a6f924..cd69003d54 100644 --- a/src/agent/onefuzz-agent/src/work.rs +++ b/src/agent/onefuzz-agent/src/work.rs @@ -30,12 +30,12 @@ impl WorkSet { self.work_units.iter().map(|w| w.task_id).collect() } - pub fn context_path() -> Result { - Ok(onefuzz::fs::onefuzz_root()?.join("workset_context.json")) + pub fn context_path(machine_id: Uuid) -> Result { + Ok(onefuzz::fs::onefuzz_root()?.join(format!("workset_context-{}.json", machine_id))) } - pub async fn load_from_fs_context() -> Result> { - let path = Self::context_path()?; + pub async fn load_from_fs_context(machine_id: Uuid) -> Result> { + let path = Self::context_path(machine_id)?; info!("checking for workset context: {}", path.display()); @@ -57,8 +57,8 @@ impl WorkSet { Ok(Some(ctx)) } - pub async fn save_context(&self) -> Result<()> { - let path = Self::context_path()?; + pub async fn save_context(&self, machine_id: Uuid) -> Result<()> { + let path = Self::context_path(machine_id)?; info!("saving workset context: {}", path.display()); let data = serde_json::to_vec(&self)?; @@ -69,6 +69,19 @@ impl WorkSet { Ok(()) } + pub async fn remove_context(machine_id: Uuid) -> Result<()> { + let path = Self::context_path(machine_id)?; + info!("removing workset context: {}", path.display()); + + if path.exists() { + fs::remove_file(&path) + .await + .with_context(|| format!("unable to delete WorkSet context: {}", path.display()))?; + } + + Ok(()) + } + pub fn setup_dir(&self) -> Result { let setup_dir = self .setup_url diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index 52b6daecff..db91fdc37c 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -37,6 +37,7 @@ pub enum WorkerEvent { }, } +#[derive(Debug)] pub enum Worker { Ready(State), Running(State), @@ -94,14 +95,17 @@ impl Worker { } } +#[derive(Debug)] pub struct Ready { setup_dir: PathBuf, } +#[derive(Debug)] pub struct Running { child: Box, } +#[derive(Debug)] pub struct Done { output: Output, } @@ -112,6 +116,7 @@ impl Context for Ready {} impl Context for Running {} impl Context for Done {} +#[derive(Debug)] pub struct State { ctx: C, work: WorkUnit, @@ -189,7 +194,7 @@ pub trait IWorkerRunner: Downcast { impl_downcast!(IWorkerRunner); -pub trait IWorkerChild: Downcast { +pub trait IWorkerChild: Downcast + std::fmt::Debug { fn try_wait(&mut self) -> Result>; fn kill(&mut self) -> Result<()>; @@ -197,6 +202,7 @@ pub trait IWorkerChild: Downcast { impl_downcast!(IWorkerChild); +#[derive(Debug)] pub struct WorkerRunner; #[async_trait] @@ -284,6 +290,7 @@ impl SuspendableChild for Child { } /// Child process with redirected output streams, tailed by two worker threads. +#[derive(Debug)] struct RedirectedChild { /// The child process. child: Child, @@ -310,6 +317,7 @@ impl RedirectedChild { } /// Worker threads that tail the redirected output streams of a running child process. +#[derive(Debug)] struct StreamReaderThreads { stderr: JoinHandle, stdout: JoinHandle, diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs index 0bc5eb8583..fa3e399b70 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs @@ -227,7 +227,10 @@ where let crash_dir = self.create_local_temp_dir().await?; let run_id = Uuid::new_v4(); - debug!("starting fuzzer run, run_id = {}", run_id); + debug!( + "starting fuzzer run, run_id = {}, worker_id = {}", + run_id, worker_id + ); let mut inputs = vec![&self.config.inputs.local_path]; if let Some(readonly_inputs) = &self.config.readonly_inputs { diff --git a/src/agent/onefuzz/src/sanitizer.rs b/src/agent/onefuzz/src/sanitizer.rs index 60e5768f52..372b8f528e 100644 --- a/src/agent/onefuzz/src/sanitizer.rs +++ b/src/agent/onefuzz/src/sanitizer.rs @@ -4,10 +4,11 @@ use std::collections::HashMap; use std::path::Path; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; pub fn default_llvm_symbolizer_path() -> Result { - Ok(std::env::var("LLVM_SYMBOLIZER_PATH")?) + Ok(std::env::var("LLVM_SYMBOLIZER_PATH") + .with_context(|| format!("LLVM_SYMBOLIZER_PATH environment variable is not set"))?) } pub fn default_sanitizer_env_vars() -> Result> { From 36aab283350b89234b2b8d11b1da19d877461bc0 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 12:03:41 -0800 Subject: [PATCH 42/52] pass the machine_identity from the agent to the task --- src/agent/onefuzz-agent/src/config.rs | 2 +- src/agent/onefuzz-agent/src/debug.rs | 8 +++++++- src/agent/onefuzz-agent/src/main.rs | 4 ++-- src/agent/onefuzz-agent/src/worker.rs | 12 ++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 5d5f65a9ea..2b583eaf35 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -150,7 +150,7 @@ impl StaticConfig { let credentials = ClientCredentials::new( client_id, - client_secret, + client_secret.into(), onefuzz_url.to_string(), tenant, multi_tenant_domain.clone(), diff --git a/src/agent/onefuzz-agent/src/debug.rs b/src/agent/onefuzz-agent/src/debug.rs index 692a0bd02b..b9b1a71fc3 100644 --- a/src/agent/onefuzz-agent/src/debug.rs +++ b/src/agent/onefuzz-agent/src/debug.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use anyhow::Result; use clap::Parser; use onefuzz::blob::BlobContainerUrl; +use onefuzz::machine_id::MachineIdentity; use onefuzz::process::ExitStatus; use url::Url; use uuid::Uuid; @@ -188,7 +189,12 @@ async fn run_worker(mut work_set: WorkSet) -> Result> { let mut worker = Worker::new(&setup_dir, work_unit); while !worker.is_done() { - worker = worker.update(&mut events, &mut WorkerRunner).await?; + worker = worker + .update( + &mut events, + &mut WorkerRunner::new(MachineIdentity::default()), + ) + .await?; } Ok(events) diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index ddb549d072..7123053b4e 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -301,7 +301,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { init_agent_heartbeat( url, config.machine_identity.machine_id, - config.machine_identity.machine_name, + config.machine_identity.machine_name.clone(), ) .await?, ), @@ -313,7 +313,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { scheduler, Box::new(setup::SetupRunner), Box::new(work_queue), - Box::new(worker::WorkerRunner), + Box::new(worker::WorkerRunner::new(config.machine_identity)), agent_heartbeat, config.managed, ); diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index 52b6daecff..0c3bce9653 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -197,7 +197,15 @@ pub trait IWorkerChild: Downcast { impl_downcast!(IWorkerChild); -pub struct WorkerRunner; +pub struct WorkerRunner { + machine_identity: MachineIdentity, +} + +impl WorkerRunner { + pub fn new(machine_identity: MachineIdentity) -> Self { + Self { machine_identity } + } +} #[async_trait] impl IWorkerRunner for WorkerRunner { @@ -221,7 +229,7 @@ impl IWorkerRunner for WorkerRunner { config.insert( "machine_identity".to_string(), - serde_json::to_value(MachineIdentity::default())?, + serde_json::to_value(&self.machine_identity)?, ); let config_path = work.config_path()?; From e710ce7fb0c241311b1e13d6d8d75d5ccaafcffd Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 12:11:02 -0800 Subject: [PATCH 43/52] clippy fix --- src/agent/onefuzz-agent/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 2b583eaf35..5d5f65a9ea 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -150,7 +150,7 @@ impl StaticConfig { let credentials = ClientCredentials::new( client_id, - client_secret.into(), + client_secret, onefuzz_url.to_string(), tenant, multi_tenant_domain.clone(), From cdde4788cb4fa37a09235b987ce28a3d7a9e6205 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 12:26:37 -0800 Subject: [PATCH 44/52] removing default from MachineIdenitty --- src/agent/onefuzz-agent/src/debug.rs | 6 +++++- src/agent/onefuzz-task/src/local/common.rs | 14 +++++++++++--- src/agent/onefuzz-task/src/local/coverage.rs | 2 +- .../onefuzz-task/src/local/generic_analysis.rs | 2 +- .../onefuzz-task/src/local/generic_crash_report.rs | 2 +- .../onefuzz-task/src/local/generic_generator.rs | 2 +- src/agent/onefuzz-task/src/local/libfuzzer.rs | 2 +- .../src/local/libfuzzer_crash_report.rs | 2 +- src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs | 2 +- .../onefuzz-task/src/local/libfuzzer_merge.rs | 2 +- .../onefuzz-task/src/local/libfuzzer_regression.rs | 2 +- .../onefuzz-task/src/local/libfuzzer_test_input.rs | 2 +- src/agent/onefuzz-task/src/local/radamsa.rs | 2 +- src/agent/onefuzz-task/src/local/test_input.rs | 2 +- src/agent/onefuzz-task/src/tasks/config.rs | 2 +- src/agent/onefuzz/src/machine_id.rs | 2 +- 16 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/agent/onefuzz-agent/src/debug.rs b/src/agent/onefuzz-agent/src/debug.rs index b9b1a71fc3..a686dc03c8 100644 --- a/src/agent/onefuzz-agent/src/debug.rs +++ b/src/agent/onefuzz-agent/src/debug.rs @@ -192,7 +192,11 @@ async fn run_worker(mut work_set: WorkSet) -> Result> { worker = worker .update( &mut events, - &mut WorkerRunner::new(MachineIdentity::default()), + &mut WorkerRunner::new(MachineIdentity { + machine_id: Uuid::new_v4(), + machine_name: "debug".into(), + scaleset_name: None, + }), ) .await?; } diff --git a/src/agent/onefuzz-task/src/local/common.rs b/src/agent/onefuzz-task/src/local/common.rs index 7472e404ca..a8252820dd 100644 --- a/src/agent/onefuzz-task/src/local/common.rs +++ b/src/agent/onefuzz-task/src/local/common.rs @@ -9,7 +9,10 @@ use anyhow::Result; use backoff::{future::retry, Error as BackoffError, ExponentialBackoff}; use clap::{App, Arg, ArgMatches}; use flume::Sender; -use onefuzz::{blob::url::BlobContainerUrl, monitor::DirectoryMonitor, syncdir::SyncedDir}; +use onefuzz::{ + blob::url::BlobContainerUrl, machine_id::MachineIdentity, monitor::DirectoryMonitor, + syncdir::SyncedDir, +}; use path_absolutize::Absolutize; use reqwest::Url; use storage_queue::{local_queue::ChannelQueueClient, QueueClient}; @@ -217,7 +220,7 @@ pub fn get_synced_dir( // fuzzing tasks from generating random task id to using UUID::nil(). This // enables making the one-shot crash report generation, which isn't really a task, // consistent across multiple runs. -pub fn build_local_context( +pub async fn build_local_context( args: &ArgMatches<'_>, generate_task_id: bool, event_sender: Option>, @@ -248,7 +251,12 @@ pub fn build_local_context( task_id, instance_id, setup_dir, - ..Default::default() + machine_identity: MachineIdentity::from_metadata().await?, + instance_telemetry_key: None, + heartbeat_queue: None, + microsoft_telemetry_key: None, + logs: None, + min_available_memory_mb: u64::default(), }; let current_dir = current_dir()?; let job_path = current_dir.join(format!("{}", job_id)); diff --git a/src/agent/onefuzz-task/src/local/coverage.rs b/src/agent/onefuzz-task/src/local/coverage.rs index 5581dcfe33..fb6bc7413d 100644 --- a/src/agent/onefuzz-task/src/local/coverage.rs +++ b/src/agent/onefuzz-task/src/local/coverage.rs @@ -63,7 +63,7 @@ pub fn build_coverage_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_coverage_config( args, false, diff --git a/src/agent/onefuzz-task/src/local/generic_analysis.rs b/src/agent/onefuzz-task/src/local/generic_analysis.rs index 8fb6530ede..4fdc7e0041 100644 --- a/src/agent/onefuzz-task/src/local/generic_analysis.rs +++ b/src/agent/onefuzz-task/src/local/generic_analysis.rs @@ -70,7 +70,7 @@ pub fn build_analysis_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_analysis_config(args, None, context.common_config.clone(), event_sender)?; run_analysis(config).await } diff --git a/src/agent/onefuzz-task/src/local/generic_crash_report.rs b/src/agent/onefuzz-task/src/local/generic_crash_report.rs index 1ab380f21a..1feb49acb9 100644 --- a/src/agent/onefuzz-task/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-task/src/local/generic_crash_report.rs @@ -79,7 +79,7 @@ pub fn build_report_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; ReportTask::new(config).managed_run().await } diff --git a/src/agent/onefuzz-task/src/local/generic_generator.rs b/src/agent/onefuzz-task/src/local/generic_generator.rs index 9e09181175..9b998f5e88 100644 --- a/src/agent/onefuzz-task/src/local/generic_generator.rs +++ b/src/agent/onefuzz-task/src/local/generic_generator.rs @@ -72,7 +72,7 @@ pub fn build_fuzz_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; GeneratorTask::new(config).run().await } diff --git a/src/agent/onefuzz-task/src/local/libfuzzer.rs b/src/agent/onefuzz-task/src/local/libfuzzer.rs index 2730625899..3cc95ce701 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer.rs @@ -34,7 +34,7 @@ use tokio::task::spawn; use uuid::Uuid; pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let crash_dir = fuzz_config .crashes diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-task/src/local/libfuzzer_crash_report.rs index acad3134de..3a6e2cd7b6 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_crash_report.rs @@ -74,7 +74,7 @@ pub fn build_report_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; ReportTask::new(config).managed_run().await } diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs index 25dd42738b..f0a5b709f3 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs @@ -58,7 +58,7 @@ pub fn build_fuzz_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; LibFuzzerFuzzTask::new(config)?.run().await } diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_merge.rs b/src/agent/onefuzz-task/src/local/libfuzzer_merge.rs index 44d5245f8d..2f650efa02 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_merge.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_merge.rs @@ -53,7 +53,7 @@ pub fn build_merge_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_merge_config(args, None, context.common_config.clone(), event_sender)?; spawn(std::sync::Arc::new(config)).await } diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_regression.rs b/src/agent/onefuzz-task/src/local/libfuzzer_regression.rs index c43a0caa03..9624900a67 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_regression.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_regression.rs @@ -74,7 +74,7 @@ pub fn build_regression_config( } pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let config = build_regression_config(args, context.common_config.clone(), event_sender)?; LibFuzzerRegressionTask::new(config).run().await } diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs index 690fa66a52..4fa03f8be4 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs @@ -14,7 +14,7 @@ use flume::Sender; use std::path::PathBuf; pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender)?; + let context = build_local_context(args, true, event_sender).await?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-task/src/local/radamsa.rs b/src/agent/onefuzz-task/src/local/radamsa.rs index 2baf18473e..14906740d3 100644 --- a/src/agent/onefuzz-task/src/local/radamsa.rs +++ b/src/agent/onefuzz-task/src/local/radamsa.rs @@ -18,7 +18,7 @@ use tokio::task::spawn; use uuid::Uuid; pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; + let context = build_local_context(args, true, event_sender.clone()).await?; let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let crash_dir = fuzz_config .crashes diff --git a/src/agent/onefuzz-task/src/local/test_input.rs b/src/agent/onefuzz-task/src/local/test_input.rs index 9544a796d1..4a775624bd 100644 --- a/src/agent/onefuzz-task/src/local/test_input.rs +++ b/src/agent/onefuzz-task/src/local/test_input.rs @@ -15,7 +15,7 @@ use flume::Sender; use std::path::PathBuf; pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, false, event_sender)?; + let context = build_local_context(args, false, event_sender).await?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-task/src/tasks/config.rs b/src/agent/onefuzz-task/src/tasks/config.rs index 1a31c39e86..6f22fbacff 100644 --- a/src/agent/onefuzz-task/src/tasks/config.rs +++ b/src/agent/onefuzz-task/src/tasks/config.rs @@ -32,7 +32,7 @@ pub enum ContainerType { Inputs, } -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Clone)] pub struct CommonConfig { pub job_id: Uuid, diff --git a/src/agent/onefuzz/src/machine_id.rs b/src/agent/onefuzz/src/machine_id.rs index f0949b28ad..a3e8f7366a 100644 --- a/src/agent/onefuzz/src/machine_id.rs +++ b/src/agent/onefuzz/src/machine_id.rs @@ -10,7 +10,7 @@ use std::time::Duration; use tokio::fs; use uuid::Uuid; -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Default, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct MachineIdentity { pub machine_id: Uuid, pub machine_name: String, From 3d3f2ff1d5f9d7aed5e957f99d6a7e9281847c40 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 12:52:53 -0800 Subject: [PATCH 45/52] fix tests --- src/agent/onefuzz/src/expand.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index 73159f80c0..aca388a54a 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -420,11 +420,19 @@ mod tests { use std::path::Path; use uuid::Uuid; + fn test_machine_identity() -> MachineIdentity { + MachineIdentity { + machine_id: Uuid::new_v4(), + machine_name: "test-machine".to_string(), + scaleset_name: None, + } + } + #[test] fn test_expand_nested() -> Result<()> { let supervisor_options = vec!["{target_options}".to_string()]; let target_options: Vec<_> = vec!["a", "b", "c"].iter().map(|p| p.to_string()).collect(); - let result = Expand::new(&MachineIdentity::default()) + let result = Expand::new(&test_machine_identity()) .target_options(&target_options) .evaluate(&supervisor_options)?; let expected = vec!["a b c"]; @@ -467,7 +475,7 @@ mod tests { let input_corpus_dir = "src"; let generated_inputs_dir = "src"; - let result = Expand::new(&MachineIdentity::default()) + let result = Expand::new(&test_machine_identity()) .input_corpus(Path::new(input_corpus_dir)) .generated_inputs(Path::new(generated_inputs_dir)) .target_options(&my_options) @@ -501,7 +509,7 @@ mod tests { ] ); - assert!(Expand::new(&MachineIdentity::default()) + assert!(Expand::new(&test_machine_identity()) .evaluate(&my_args) .is_err()); @@ -510,7 +518,7 @@ mod tests { #[test] fn test_expand_in_string() -> Result<()> { - let result = Expand::new(&MachineIdentity::default()) + let result = Expand::new(&test_machine_identity()) .input_path("src/lib.rs") .evaluate_value("a {input} b")?; assert!(result.contains("lib.rs")); @@ -520,10 +528,7 @@ mod tests { #[tokio::test] async fn test_expand_machine_id() -> Result<()> { let machine_id = Uuid::new_v4(); - let machine_identity = MachineIdentity { - machine_id, - ..Default::default() - }; + let machine_identity = &test_machine_identity(); let expand = Expand::new(&machine_identity).machine_id().await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. From 83ca26c64e6fa54095a4b842180ba3095bb2ef17 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 13:00:13 -0800 Subject: [PATCH 46/52] more tests fixes --- src/agent/onefuzz/examples/test-input.rs | 6 +++++- src/agent/onefuzz/src/libfuzzer.rs | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/agent/onefuzz/examples/test-input.rs b/src/agent/onefuzz/examples/test-input.rs index 3f83df9107..bb0af41230 100644 --- a/src/agent/onefuzz/examples/test-input.rs +++ b/src/agent/onefuzz/examples/test-input.rs @@ -58,7 +58,11 @@ async fn main() -> Result<()> { &opt.exe, &target_options, &env, - MachineIdentity::default(), + MachineIdentity{ + machine_id: uuid::Uuid::new_v4(), + machine_name: "test-input".into(), + scaleset_name: None + }, ); let check_debugger = !opt.no_check_debugger; diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index ec6aeec262..9f8239ec5f 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -445,7 +445,11 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), - MachineIdentity::default(), + MachineIdentity{ + machine_id: uuid::Uuid::new_v4(), + machine_name: "test-input".into(), + scaleset_name: None + }, ); // verify catching bad exits with -help=1 @@ -474,7 +478,11 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), - MachineIdentity::default(), + MachineIdentity{ + machine_id: uuid::Uuid::new_v4(), + machine_name: "test-input".into(), + scaleset_name: None + }, ); // verify good exits with -help=1 assert!( From 8fd1fcfe4d0242607645c3a400c26e73e7c9be06 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 13:02:48 -0800 Subject: [PATCH 47/52] format --- src/agent/onefuzz/examples/test-input.rs | 4 ++-- src/agent/onefuzz/src/libfuzzer.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz/examples/test-input.rs b/src/agent/onefuzz/examples/test-input.rs index bb0af41230..aff2ae86a8 100644 --- a/src/agent/onefuzz/examples/test-input.rs +++ b/src/agent/onefuzz/examples/test-input.rs @@ -58,10 +58,10 @@ async fn main() -> Result<()> { &opt.exe, &target_options, &env, - MachineIdentity{ + MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test-input".into(), - scaleset_name: None + scaleset_name: None, }, ); diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 9f8239ec5f..b20315d857 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -445,10 +445,10 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), - MachineIdentity{ + MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test-input".into(), - scaleset_name: None + scaleset_name: None, }, ); @@ -478,10 +478,10 @@ mod tests { options.clone(), env.clone(), &temp_setup_dir.path(), - MachineIdentity{ + MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test-input".into(), - scaleset_name: None + scaleset_name: None, }, ); // verify good exits with -help=1 From 61196646dcc9e42161c912ad06cd8e1a8c4f5eee Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 13:48:53 -0800 Subject: [PATCH 48/52] clippy fix --- .../onefuzz-task/src/tasks/fuzz/generator.rs | 17 ++++++++++++++++- .../onefuzz-task/src/tasks/fuzz/supervisor.rs | 18 +++++++++++++++++- src/agent/onefuzz/src/expand.rs | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs index a9dad3e48c..1ca255f661 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs @@ -289,7 +289,22 @@ mod tests { ensemble_sync_delay: None, generator_env: HashMap::default(), check_retry_count: 0, - common: CommonConfig::default(), + common: CommonConfig { + job_id: Default::default(), + task_id: Default::default(), + instance_id: Default::default(), + heartbeat_queue: Default::default(), + instance_telemetry_key: Default::default(), + microsoft_telemetry_key: Default::default(), + logs: Default::default(), + setup_dir: Default::default(), + min_available_memory_mb: Default::default(), + machine_identity: MachineIdentity { + machine_id: uuid::Uuid::new_v4(), + machine_name: "test".to_string(), + scaleset_name: None, + }, + }, }; let task = GeneratorTask::new(config); diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs index df71249c09..ee28640120 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs @@ -279,6 +279,7 @@ mod tests { use super::*; use crate::tasks::stats::afl::read_stats; use onefuzz::blob::BlobContainerUrl; + use onefuzz::machine_id::MachineIdentity; use onefuzz::process::monitor_process; use onefuzz_telemetry::EventData; use reqwest::Url; @@ -381,7 +382,22 @@ mod tests { unique_reports: None, no_repro: None, coverage: None, - common: CommonConfig::default(), + common: CommonConfig { + job_id: Default::default(), + task_id: Default::default(), + instance_id: Default::default(), + heartbeat_queue: Default::default(), + instance_telemetry_key: Default::default(), + microsoft_telemetry_key: Default::default(), + logs: Default::default(), + setup_dir: Default::default(), + min_available_memory_mb: Default::default(), + machine_identity: MachineIdentity { + machine_id: uuid::Uuid::new_v4(), + machine_name: "test".to_string(), + scaleset_name: None, + }, + }, }; let process = start_supervisor(runtime_dir, &config, &crashes, &corpus_dir, reports_dir) diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index aca388a54a..d04cfdce6a 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -529,7 +529,7 @@ mod tests { async fn test_expand_machine_id() -> Result<()> { let machine_id = Uuid::new_v4(); let machine_identity = &test_machine_identity(); - let expand = Expand::new(&machine_identity).machine_id().await?; + let expand = Expand::new(machine_identity).machine_id().await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. let expanded_machine_id = Uuid::parse_str(&expanded)?; From 5dfcfb5d7f97edab0160aa9a201dc064158bc58c Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 13:54:39 -0800 Subject: [PATCH 49/52] another fix --- src/agent/onefuzz-task/src/tasks/fuzz/generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs index 1ca255f661..c44927db10 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs @@ -299,7 +299,7 @@ mod tests { logs: Default::default(), setup_dir: Default::default(), min_available_memory_mb: Default::default(), - machine_identity: MachineIdentity { + machine_identity: onefuzz::machine_id::MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test".to_string(), scaleset_name: None, From 64811da75322477b2467506819a9245b6319e150 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 14:11:09 -0800 Subject: [PATCH 50/52] fix test --- src/agent/onefuzz/src/expand.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index d04cfdce6a..d35cfce50d 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -527,8 +527,8 @@ mod tests { #[tokio::test] async fn test_expand_machine_id() -> Result<()> { - let machine_id = Uuid::new_v4(); let machine_identity = &test_machine_identity(); + let machine_id = machine_identity.machine_id; let expand = Expand::new(machine_identity).machine_id().await?; let expanded = expand.evaluate_value("{machine_id}")?; // Check that "{machine_id}" expands to a valid UUID, but don't worry about the actual value. From 828b1a2c9884517b1805ebbc08e1012e0855ff16 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 30 Nov 2022 18:28:10 -0800 Subject: [PATCH 51/52] more fixes includin the machine in the path of the working di --- src/agent/onefuzz-agent/src/scheduler.rs | 2 +- src/agent/onefuzz-agent/src/setup.rs | 12 ++++++++---- src/agent/onefuzz-agent/src/work.rs | 10 ++++++---- src/agent/onefuzz-agent/src/worker.rs | 4 ++-- src/agent/onefuzz-task/src/local/libfuzzer.rs | 8 ++++++++ 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/agent/onefuzz-agent/src/scheduler.rs b/src/agent/onefuzz-agent/src/scheduler.rs index e786d9fa82..3fc8403f4f 100644 --- a/src/agent/onefuzz-agent/src/scheduler.rs +++ b/src/agent/onefuzz-agent/src/scheduler.rs @@ -209,7 +209,7 @@ impl State { // No script was executed. } Err(err) => { - let error = format!("{:?}", err.to_string()); + let error = format!("{:?}", err); warn!("{}", error); let cause = DoneCause::SetupError { error, diff --git a/src/agent/onefuzz-agent/src/setup.rs b/src/agent/onefuzz-agent/src/setup.rs index 1a2f046045..0e4fc4a9e1 100644 --- a/src/agent/onefuzz-agent/src/setup.rs +++ b/src/agent/onefuzz-agent/src/setup.rs @@ -64,7 +64,7 @@ impl SetupRunner { // Create setup container directory symlinks for tasks. for work_unit in &work_set.work_units { - create_setup_symlink(&setup_dir, work_unit).await?; + create_setup_symlink(&setup_dir, work_unit, self.machine_id).await?; } // Run setup script, if any. @@ -103,13 +103,17 @@ impl SetupRunner { } #[cfg(target_family = "windows")] -async fn create_setup_symlink(setup_dir: &Path, work_unit: &WorkUnit) -> Result<()> { +async fn create_setup_symlink( + setup_dir: &Path, + work_unit: &WorkUnit, + machine_id: Uuid, +) -> Result<()> { use std::os::windows::fs::symlink_dir; use tokio::task::spawn_blocking; - let working_dir = work_unit.working_dir()?; + let working_dir = work_unit.working_dir(machine_id)?; - let create_work_dir = fs::create_dir(&working_dir).await.with_context(|| { + let create_work_dir = fs::create_dir_all(&working_dir).await.with_context(|| { format!( "unable to create working directory: {}", working_dir.display() diff --git a/src/agent/onefuzz-agent/src/work.rs b/src/agent/onefuzz-agent/src/work.rs index cd69003d54..34b93af494 100644 --- a/src/agent/onefuzz-agent/src/work.rs +++ b/src/agent/onefuzz-agent/src/work.rs @@ -106,12 +106,14 @@ pub struct WorkUnit { } impl WorkUnit { - pub fn working_dir(&self) -> Result { - Ok(onefuzz::fs::onefuzz_root()?.join(self.task_id.to_string())) + pub fn working_dir(&self, machine_id: Uuid) -> Result { + Ok(onefuzz::fs::onefuzz_root()? + .join(format!("{}", machine_id)) + .join(self.task_id.to_string())) } - pub fn config_path(&self) -> Result { - Ok(self.working_dir()?.join("config.json")) + pub fn config_path(&self, machine_id: Uuid) -> Result { + Ok(self.working_dir(machine_id)?.join("config.json")) } } diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index 06c2da4588..65492874d3 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -215,7 +215,7 @@ impl WorkerRunner { #[async_trait] impl IWorkerRunner for WorkerRunner { async fn run(&mut self, setup_dir: &Path, work: &WorkUnit) -> Result> { - let working_dir = work.working_dir()?; + let working_dir = work.working_dir(self.machine_identity.machine_id)?; debug!("worker working dir = {}", working_dir.display()); @@ -237,7 +237,7 @@ impl IWorkerRunner for WorkerRunner { serde_json::to_value(&self.machine_identity)?, ); - let config_path = work.config_path()?; + let config_path = work.config_path(self.machine_identity.machine_id)?; fs::write(&config_path, serde_json::to_string(&config)?.as_bytes()) .await diff --git a/src/agent/onefuzz-task/src/local/libfuzzer.rs b/src/agent/onefuzz-task/src/local/libfuzzer.rs index 3cc95ce701..ecd3e3450d 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer.rs @@ -87,6 +87,14 @@ pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option Date: Mon, 5 Dec 2022 16:38:49 -0800 Subject: [PATCH 52/52] rename client_id in pool to object_id --- .../ApiService/Functions/AgentRegistration.cs | 2 +- src/ApiService/ApiService/Functions/Pool.cs | 4 +- .../ApiService/OneFuzzTypes/Model.cs | 2 +- .../ApiService/OneFuzzTypes/Requests.cs | 2 +- .../ApiService/OneFuzzTypes/Responses.cs | 2 +- .../onefuzzlib/EndpointAuthorization.cs | 40 +++++++++---------- .../ApiService/onefuzzlib/PoolOperations.cs | 12 +++--- src/agent/.gitignore | 1 + src/cli/onefuzz/api.py | 4 +- src/pytypes/onefuzztypes/requests.py | 2 +- 10 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index c35871a5d2..e858197293 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -71,7 +71,7 @@ private async Async.Task Get(HttpRequestData req) { private async Async.Task CreateRegistrationResponse(Service.Pool pool) { var hostName = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME"); - var scheme = Environment.GetEnvironmentVariable("HTTPS") != null? "https" : "http"; + var scheme = Environment.GetEnvironmentVariable("HTTPS") != null ? "https" : "http"; var baseAddress = $"{scheme}://{hostName}"; var eventsUrl = new Uri($"{baseAddress}/api/agents/events"); var commandsUrl = new Uri($"{baseAddress}/api/agents/commands"); diff --git a/src/ApiService/ApiService/Functions/Pool.cs b/src/ApiService/ApiService/Functions/Pool.cs index 6ca19ec640..497a8fd2c8 100644 --- a/src/ApiService/ApiService/Functions/Pool.cs +++ b/src/ApiService/ApiService/Functions/Pool.cs @@ -67,7 +67,7 @@ private async Task Post(HttpRequestData req) { Errors: new string[] { "pool with that name already exists" }), "PoolCreate"); } - var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, clientId: create.ClientId); + var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, objectId: create.ObjectId); return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(newPool), true)); } @@ -106,7 +106,7 @@ private static PoolGetResult PoolToPoolResponse(Service.Pool p) PoolId: p.PoolId, Os: p.Os, State: p.State, - ClientId: p.ClientId, + ObjectId: p.ObjectId, Managed: p.Managed, Arch: p.Arch, Nodes: p.Nodes, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 5f9063561a..382b6b9542 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -643,7 +643,7 @@ public record Pool( bool Managed, Architecture Arch, PoolState State, - Guid? ClientId = null + Guid? ObjectId = null ) : StatefulEntityBase(State) { public List? Nodes { get; set; } public AgentConfig? Config { get; set; } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 2ca98016a4..4037c9bde9 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -263,7 +263,7 @@ public record PoolCreate( [property: Required] Os Os, [property: Required] Architecture Arch, [property: Required] bool Managed, - Guid? ClientId = null + Guid? ObjectId = null ) : BaseRequest; public record WebhookCreate( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index a09be7d061..d2da18b64b 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -114,7 +114,7 @@ public record PoolGetResult( bool Managed, Architecture Arch, PoolState State, - Guid? ClientId, + Guid? ObjectId, List? Nodes, AgentConfig? Config, List? WorkQueue, diff --git a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs index b302819b5b..5350bc23f7 100644 --- a/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs +++ b/src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs @@ -46,9 +46,12 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu } var token = tokenResult.OkV.UserInfo; - if (await IsUser(tokenResult.OkV)) { + + var (isAgent, reason) = await IsAgent(tokenResult.OkV); + + if (!isAgent) { if (!allowUser) { - return await Reject(req, token); + return await Reject(req, token, "endpoint not allowed for users"); } var access = await CheckAccess(req); @@ -57,21 +60,18 @@ public virtual async Async.Task CallIf(HttpRequestData req, Fu } } - if (await IsAgent(tokenResult.OkV) && !allowAgent) { + if (isAgent && !allowAgent) { - return await Reject(req, token); + return await Reject(req, token, reason); } return await method(req); } - public async Async.Task IsUser(UserAuthInfo tokenData) { - return !await IsAgent(tokenData); - } - public async Async.Task Reject(HttpRequestData req, UserInfo token) { + public async Async.Task Reject(HttpRequestData req, UserInfo token, String? reason = null) { var body = await req.ReadAsStringAsync(); - _log.Error($"reject token. url:{req.Url:Tag:Url} token:{token:Tag:Token} body:{body:Tag:Body}"); + _log.Error($"reject token. reason:{reason} url:{req.Url:Tag:Url} token:{token:Tag:Token} body:{body:Tag:Body}"); return await _context.RequestHandling.NotOk( req, @@ -188,9 +188,9 @@ private GroupMembershipChecker CreateGroupMembershipChecker(InstanceConfig confi } - public async Async.Task IsAgent(UserAuthInfo authInfo) { + public async Async.Task<(bool, string)> IsAgent(UserAuthInfo authInfo) { if (!AgentRoles.Overlaps(authInfo.Roles)) { - return false; + return (false, "no agent role"); } var tokenData = authInfo.UserInfo; @@ -198,28 +198,24 @@ public async Async.Task IsAgent(UserAuthInfo authInfo) { if (tokenData.ObjectId != null) { var scalesets = _context.ScalesetOperations.GetByObjectId(tokenData.ObjectId.Value); if (await scalesets.AnyAsync()) { - return true; + return (true, string.Empty); } var principalId = await _context.Creds.GetScalesetPrincipalId(); if (principalId == tokenData.ObjectId) { - return true; + return (true, string.Empty); } } - if (!tokenData.ApplicationId.HasValue) { - return false; + if (!tokenData.ObjectId.HasValue) { + return (false, "no object id in token"); } - var pools = _context.PoolOperations.GetByClientId(tokenData.ApplicationId.Value); + var pools = _context.PoolOperations.GetByObjectId(tokenData.ObjectId.Value); if (await pools.AnyAsync()) { - return true; + return (true, string.Empty); } - // if (tokenData.Roles.Contains("unmanagedNode")) { - // return true; - // } - - return false; + return (false, "no matching scaleset or pool"); } } diff --git a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs index 09c0ec342c..9a3662f3b6 100644 --- a/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/PoolOperations.cs @@ -6,14 +6,14 @@ public interface IPoolOperations : IStatefulOrm { Async.Task> GetByName(PoolName poolName); Async.Task> GetById(Guid poolId); Task ScheduleWorkset(Pool pool, WorkSet workSet); - IAsyncEnumerable GetByClientId(Guid clientId); + IAsyncEnumerable GetByObjectId(Guid objectId); string GetPoolQueue(Guid poolId); Async.Task> GetScalesetSummary(PoolName name); Async.Task> GetWorkQueue(Guid poolId, PoolState state); IAsyncEnumerable SearchStates(IEnumerable states); Async.Task SetShutdown(Pool pool, bool Now); - Async.Task Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null); + Async.Task Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? objectId = null); new Async.Task Delete(Pool pool); // state transitions: @@ -32,7 +32,7 @@ public PoolOperations(ILogTracer log, IOnefuzzContext context) } - public async Async.Task Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null) { + public async Async.Task Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? objectId = null) { var newPool = new Service.Pool( PoolId: Guid.NewGuid(), State: PoolState.Init, @@ -40,7 +40,7 @@ public async Async.Task Create(PoolName name, Os os, Architecture architec Os: os, Managed: managed, Arch: architecture, - ClientId: clientId); + ObjectId: objectId); var r = await Insert(newPool); if (!r.IsOk) { @@ -87,8 +87,8 @@ public async Task ScheduleWorkset(Pool pool, WorkSet workSet) { return await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus); } - public IAsyncEnumerable GetByClientId(Guid clientId) { - return QueryAsync(filter: $"client_id eq '{clientId}'"); + public IAsyncEnumerable GetByObjectId(Guid objectId) { + return QueryAsync(filter: $"object_id eq '{objectId}'"); } public string GetPoolQueue(Guid poolId) diff --git a/src/agent/.gitignore b/src/agent/.gitignore index eb5a316cbd..8769e3d5f1 100644 --- a/src/agent/.gitignore +++ b/src/agent/.gitignore @@ -1 +1,2 @@ target +.agent-run \ No newline at end of file diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 51aad410b2..f7659356d7 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1239,7 +1239,7 @@ def create( self, name: str, os: enums.OS, - client_id: Optional[UUID] = None, + object_id: Optional[UUID] = None, *, unmanaged: bool = False, arch: enums.Architecture = enums.Architecture.x86_64, @@ -1256,7 +1256,7 @@ def create( "POST", models.Pool, data=requests.PoolCreate( - name=name, os=os, arch=arch, managed=managed, client_id=client_id + name=name, os=os, arch=arch, managed=managed, object_id=object_id ), ) diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 54bad0adec..1bdf37dff3 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -92,7 +92,7 @@ class PoolCreate(BaseRequest): os: OS arch: Architecture managed: bool - client_id: Optional[UUID] + object_id: Optional[UUID] autoscale: Optional[AutoScaleConfig]