[WIP] unmanaged node prototype

contrib/deploy-agent-in-docker/linux/Dockerfile

@@ -0,0 +1,27 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04
+
+# Pull Request that contains OneFuzz release-artifacts
+# used to create the Docker container
+ARG RELEASE_VERSION
+
+RUN apt-get update
+RUN apt-get install --yes --quiet curl \
+        unzip \
+        wget
+
+# creating a dummy sudo command
+RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
+
+RUN wget https://github.com/microsoft/onefuzz/releases/download/${RELEASE_VERSION}/onefuzz-deployment-${RELEASE_VERSION}.zip && \
+    unzip -j onefuzz-deployment-${RELEASE_VERSION}.zip tools/linux/* -d onefuzz-install/ && \
+    cd onefuzz-install && \
+    chmod +x ./onefuzz-agent && \
+    chmod +x ./run.sh && \
+    chmod +x ./setup.sh
+
+RUN cd onefuzz-install && ./setup.sh
+
+ENTRYPOINT onefuzz/run.sh

src/ApiService/ApiService/Functions/AgentRegistration.cs

@@ -70,9 +70,11 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
     }
 
     private async Async.Task<AgentRegistrationResponse> CreateRegistrationResponse(Service.Pool pool) {
-        var baseAddress = _context.Creds.GetInstanceUrl();
-        var eventsUrl = new Uri(baseAddress, "/api/agents/events");
-        var commandsUrl = new Uri(baseAddress, "/api/agents/commands");
+        var hostName = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME");
+        var scheme = Environment.GetEnvironmentVariable("HTTPS") != null ? "https" : "http";
+        var baseAddress = $"{scheme}://{hostName}";
+        var eventsUrl = new Uri($"{baseAddress}/api/agents/events");
+        var commandsUrl = new Uri($"{baseAddress}/api/agents/commands");
 
         var workQueue = await _context.Queue.GetQueueSas(
             _context.PoolOperations.GetPoolQueue(pool.PoolId),
@@ -86,6 +88,7 @@ private async Async.Task<AgentRegistrationResponse> CreateRegistrationResponse(S
             WorkQueue: workQueue);
     }
 
+    // todo: add agent registration post
     private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
         var request = await RequestHandling.ParseUri<AgentRegistrationPost>(req);
         if (!request.IsOk) {
@@ -117,6 +120,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
                 "agent registration");
         }
 
+
         var instanceId = machineName is not null ? InstanceIds.InstanceIdFromMachineName(machineName) : null;
 
 
@@ -153,7 +157,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
             MachineId: machineId,
             ScalesetId: scalesetId,
             InstanceId: instanceId,
-            Version: version
+            Version: version,
+            Os: os ?? pool.Os,
+            Managed: pool.Managed
             );
 
         var r = await _context.NodeOperations.Replace(node);

src/ApiService/ApiService/Functions/Pool.cs

@@ -67,7 +67,7 @@ private async Task<HttpResponseData> Post(HttpRequestData req) {
                     Errors: new string[] { "pool with that name already exists" }),
                 "PoolCreate");
         }
-        var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, clientId: create.ClientId);
+        var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, objectId: create.ObjectId);
         return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(newPool), true));
     }
 
@@ -106,7 +106,7 @@ private static PoolGetResult PoolToPoolResponse(Service.Pool p)
             PoolId: p.PoolId,
             Os: p.Os,
             State: p.State,
-            ClientId: p.ClientId,
+            ObjectId: p.ObjectId,
             Managed: p.Managed,
             Arch: p.Arch,
             Nodes: p.Nodes,

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -110,7 +110,8 @@ public record Node
 
     bool ReimageRequested = false,
     bool DeleteRequested = false,
-    bool DebugKeepNode = false
+    bool DebugKeepNode = false,
+    bool Managed = true
 ) : StatefulEntityBase<NodeState>(State) {
 
     public List<NodeTasks>? Tasks { get; set; }
@@ -642,7 +643,7 @@ public record Pool(
     bool Managed,
     Architecture Arch,
     PoolState State,
-    Guid? ClientId = null
+    Guid? ObjectId = null
 ) : StatefulEntityBase<PoolState>(State) {
     public List<Node>? Nodes { get; set; }
     public AgentConfig? Config { get; set; }

src/ApiService/ApiService/OneFuzzTypes/Requests.cs

@@ -263,7 +263,7 @@ public record PoolCreate(
     [property: Required] Os Os,
     [property: Required] Architecture Arch,
     [property: Required] bool Managed,
-    Guid? ClientId = null
+    Guid? ObjectId = null
 ) : BaseRequest;
 
 public record WebhookCreate(

src/ApiService/ApiService/OneFuzzTypes/Responses.cs

@@ -114,11 +114,12 @@ public record PoolGetResult(
     bool Managed,
     Architecture Arch,
     PoolState State,
-    Guid? ClientId,
+    Guid? ObjectId,
     List<Node>? Nodes,
     AgentConfig? Config,
     List<WorkSetSummary>? WorkQueue,
     List<ScalesetSummary>? ScalesetSummary
+//List<Node>? UnmanagedNodes
 ) : BaseResponse();
 
 public record ScalesetResponse(

src/ApiService/ApiService/UserCredentials.cs

@@ -71,6 +71,7 @@ public virtual async Task<OneFuzzResult<UserAuthInfo>> ParseJwtToken(HttpRequest
             var allowedTenants = await GetAllowedTenants();
             if (allowedTenants.IsOk) {
                 if (allowedTenants.OkV is not null && allowedTenants.OkV.Contains(token.Issuer)) {
+
                     var userAuthInfo = new UserAuthInfo(new UserInfo(null, null, null), new List<string>());
                     var userInfo =
                         token.Payload.Claims.Aggregate(userAuthInfo, (acc, claim) => {

src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs

@@ -46,9 +46,12 @@ public virtual async Async.Task<HttpResponseData> CallIf(HttpRequestData req, Fu
         }
 
         var token = tokenResult.OkV.UserInfo;
-        if (await IsUser(tokenResult.OkV)) {
+
+        var (isAgent, reason) = await IsAgent(tokenResult.OkV);
+
+        if (!isAgent) {
             if (!allowUser) {
-                return await Reject(req, token);
+                return await Reject(req, token, "endpoint not allowed for users");
             }
 
             var access = await CheckAccess(req);
@@ -57,20 +60,18 @@ public virtual async Async.Task<HttpResponseData> CallIf(HttpRequestData req, Fu
             }
         }
 
-        if (await IsAgent(tokenResult.OkV) && !allowAgent) {
-            return await Reject(req, token);
+        if (isAgent && !allowAgent) {
+
+            return await Reject(req, token, reason);
         }
 
         return await method(req);
     }
 
-    public async Async.Task<bool> IsUser(UserAuthInfo tokenData) {
-        return !await IsAgent(tokenData);
-    }
 
-    public async Async.Task<HttpResponseData> Reject(HttpRequestData req, UserInfo token) {
+    public async Async.Task<HttpResponseData> Reject(HttpRequestData req, UserInfo token, String? reason = null) {
         var body = await req.ReadAsStringAsync();
-        _log.Error($"reject token. url:{req.Url:Tag:Url} token:{token:Tag:Token} body:{body:Tag:Body}");
+        _log.Error($"reject token. reason:{reason} url:{req.Url:Tag:Url} token:{token:Tag:Token} body:{body:Tag:Body}");
 
         return await _context.RequestHandling.NotOk(
             req,
@@ -186,34 +187,35 @@ private GroupMembershipChecker CreateGroupMembershipChecker(InstanceConfig confi
         return null;
     }
 
-    public async Async.Task<bool> IsAgent(UserAuthInfo authInfo) {
+
+    public async Async.Task<(bool, string)> IsAgent(UserAuthInfo authInfo) {
         if (!AgentRoles.Overlaps(authInfo.Roles)) {
-            return false;
+            return (false, "no agent role");
         }
 
         var tokenData = authInfo.UserInfo;
 
         if (tokenData.ObjectId != null) {
             var scalesets = _context.ScalesetOperations.GetByObjectId(tokenData.ObjectId.Value);
             if (await scalesets.AnyAsync()) {
-                return true;
+                return (true, string.Empty);
             }
 
             var principalId = await _context.Creds.GetScalesetPrincipalId();
             if (principalId == tokenData.ObjectId) {
-                return true;
+                return (true, string.Empty);
             }
         }
 
-        if (!tokenData.ApplicationId.HasValue) {
-            return false;
+        if (!tokenData.ObjectId.HasValue) {
+            return (false, "no object id in token");
         }
 
-        var pools = _context.PoolOperations.GetByClientId(tokenData.ApplicationId.Value);
+        var pools = _context.PoolOperations.GetByObjectId(tokenData.ObjectId.Value);
         if (await pools.AnyAsync()) {
-            return true;
+            return (true, string.Empty);
         }
 
-        return false;
+        return (false, "no matching scaleset or pool");
     }
 }

src/ApiService/ApiService/onefuzzlib/Extension.cs

@@ -12,6 +12,9 @@ public interface IExtensions {
     Async.Task<IList<VirtualMachineScaleSetExtensionData>> FuzzExtensions(Pool pool, Scaleset scaleset);
 
     Async.Task<Dictionary<string, VirtualMachineExtensionData>> ReproExtensions(AzureLocation region, Os reproOs, Guid reproId, ReproConfig reproConfig, Container? setupContainer);
+
+    Async.Task<Uri?> BuildPoolConfig(Pool pool);
+    Task<AgentConfig> CreatePoolConfig(Pool pool);
     Task<IList<VMExtensionWrapper>> ProxyManagerExtensions(Region region, Guid proxyId);
 }
 
@@ -209,6 +212,15 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) {
 
 
     public async Async.Task<Uri?> BuildPoolConfig(Pool pool) {
+        var config = await CreatePoolConfig(pool);
+
+        var fileName = $"{pool.Name}/config.json";
+        var configJson = JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions());
+        await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, fileName, configJson, StorageType.Config);
+        return await ConfigUrl(WellKnownContainers.VmScripts, fileName, false);
+    }
+
+    public async Task<AgentConfig> CreatePoolConfig(Pool pool) {
         var instanceId = await _context.Containers.GetInstanceId();
 
         var queueSas = await _context.Queue.GetQueueSas("node-heartbeat", StorageType.Config, QueueSasPermissions.Add);
@@ -222,12 +234,10 @@ public static VMExtensionWrapper GenevaExtension(AzureLocation region) {
             MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain,
             InstanceId: instanceId,
             Managed: pool.Managed
-            );
+        );
 
-        var fileName = $"{pool.Name}/config.json";
-        var configJson = JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions());
-        await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, fileName, configJson, StorageType.Config);
-        return await ConfigUrl(WellKnownContainers.VmScripts, fileName, false);
+
+        return config;
     }
 
 

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -337,6 +337,10 @@ public async Async.Task CleanupBusyNodesWithoutWork() {
     }
 
     public async Async.Task<Node> ToReimage(Node node, bool done = false) {
+        if (!node.Managed) {
+            _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}");
+            return node;
+        }
 
         var nodeState = node.State;
         if (done) {

src/ApiService/ApiService/onefuzzlib/PoolOperations.cs

@@ -6,14 +6,14 @@ public interface IPoolOperations : IStatefulOrm<Pool, PoolState> {
     Async.Task<OneFuzzResult<Pool>> GetByName(PoolName poolName);
     Async.Task<OneFuzzResult<Pool>> GetById(Guid poolId);
     Task<bool> ScheduleWorkset(Pool pool, WorkSet workSet);
-    IAsyncEnumerable<Pool> GetByClientId(Guid clientId);
+    IAsyncEnumerable<Pool> GetByObjectId(Guid objectId);
     string GetPoolQueue(Guid poolId);
     Async.Task<List<ScalesetSummary>> GetScalesetSummary(PoolName name);
     Async.Task<List<WorkSetSummary>> GetWorkQueue(Guid poolId, PoolState state);
     IAsyncEnumerable<Pool> SearchStates(IEnumerable<PoolState> states);
     Async.Task<Pool> SetShutdown(Pool pool, bool Now);
 
-    Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null);
+    Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? objectId = null);
     new Async.Task Delete(Pool pool);
 
     // state transitions:
@@ -32,15 +32,15 @@ public PoolOperations(ILogTracer log, IOnefuzzContext context)
 
     }
 
-    public async Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null) {
+    public async Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? objectId = null) {
         var newPool = new Service.Pool(
         PoolId: Guid.NewGuid(),
         State: PoolState.Init,
         Name: name,
         Os: os,
         Managed: managed,
         Arch: architecture,
-        ClientId: clientId);
+        ObjectId: objectId);
 
         var r = await Insert(newPool);
         if (!r.IsOk) {
@@ -87,8 +87,8 @@ public async Task<bool> ScheduleWorkset(Pool pool, WorkSet workSet) {
         return await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus);
     }
 
-    public IAsyncEnumerable<Pool> GetByClientId(Guid clientId) {
-        return QueryAsync(filter: $"client_id eq '{clientId}'");
+    public IAsyncEnumerable<Pool> GetByObjectId(Guid objectId) {
+        return QueryAsync(filter: $"object_id eq '{objectId}'");
     }
 
     public string GetPoolQueue(Guid poolId)

src/ApiService/IntegrationTests/ScalesetTests.cs

@@ -70,7 +70,7 @@ public async Async.Task Search_AllScalesets_ReturnsEmptyIfNoneFound() {
     public async Async.Task Create_Scaleset() {
         var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
 
-        // override the found user credentials 
+        // override the found user credentials
         var userObjectId = Guid.NewGuid();
         var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: userObjectId, "upn");
         Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));

src/ApiService/Tests/RequestsTests.cs

@@ -76,7 +76,7 @@ public void EnsureRequiredAttributesExistsOnNonNullableRequestProperties(Type re
             // find appropriate parameter
             var param = requestType.GetConstructors().Single().GetParameters().Single(p => p.Name == property.Name);
             Assert.True(param.HasDefaultValue,
-                "For request types, all non-nullable properties should either have a default value, or the [Required] attribute."
+                $"type '{requestType}' is invalid. For request types, all non-nullable properties should either have a default value, or the [Required] attribute."
             );
         } else {
             // it is required, okay

src/agent/.gitignore

@@ -1 +1,2 @@
 target
+.agent-run

src/agent/onefuzz-agent/src/agent.rs

@@ -28,6 +28,7 @@ pub struct Agent {
     previous_state: NodeState,
     last_poll_command: Result<Option<NodeCommand>, PollCommandError>,
     managed: bool,
+    machine_id: uuid::Uuid,
 }
 
 impl Agent {
@@ -40,6 +41,7 @@ impl Agent {
         worker_runner: Box<dyn IWorkerRunner>,
         heartbeat: Option<AgentHeartbeatClient>,
         managed: bool,
+        machine_id: uuid::Uuid,
     ) -> Self {
         let scheduler = Some(scheduler);
         let previous_state = NodeState::Init;
@@ -56,6 +58,7 @@ impl Agent {
             previous_state,
             last_poll_command,
             managed,
+            machine_id,
         }
     }
 
@@ -266,7 +269,7 @@ impl Agent {
 
     async fn done(&mut self, state: State<Done>) -> Result<Scheduler> {
         debug!("agent done");
-        set_done_lock().await?;
+        set_done_lock(self.machine_id).await?;
 
         let event = match state.cause() {
             DoneCause::SetupError {