add basic package detection using GPT4o mini

HomeAutomation.Web/Controllers/PackageController.cs

@@ -0,0 +1,73 @@
+using HomeAutomation.Web.Data;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
+using OpenAI_API;
+using OpenAI_API.Chat;
+
+namespace HomeAutomation.Web.Controllers;
+
+[Route("api/[controller]")]
+public class PackageController : BaseController
+{
+    private readonly PackageOptions _options;
+    private readonly HttpClient _httpClient;
+
+    public PackageController(IOptions<PackageOptions> options, HttpClient httpClient)
+    {
+        _options = options.Value;
+        _httpClient = httpClient;
+    }
+
+    [HttpGet]
+    public async Task<PackageResponse> GetPackage()
+    {
+        var openAI = new OpenAIAPI(new APIAuthentication(_options.OpenAIApiKey));
+
+        foreach (var url in _options.ImageUrls)
+        {
+            var image = await _httpClient.GetAsync(url);
+            var imageBytes = await image.Content.ReadAsByteArrayAsync();
+
+            var chatRequest = new ChatRequest
+            {
+                Model = "gpt-4o-mini",
+                Temperature = 0,
+                MaxTokens = 10,
+                FrequencyPenalty = 0,
+                PresencePenalty = 0,
+                Messages =
+                [
+                    new ChatMessage
+                    {
+                        Role = ChatMessageRole.System,
+                        TextContent = @"
+You are an AI assistant designed to help the user determine if there is a parcel left on their doorstep. They will upload a photo of a top down view from a CCTV camera which may be in colour or black and white and your job is to return only the word 'true' or 'false' depending on if you see a parcel or not.
+
+The parcel may take any shape or form, such as a box, letter and it may be any size. You are to ignore static objects in the scene such as doorbells, the door itself, cars, people, doormats and other items that are not parcels.
+
+Do not say anything except 'true' or 'false', the output will be processed by code.
+",
+
+                    },
+                    new() {
+                        Role = ChatMessageRole.User,
+                        Images =
+                        [
+                            new ChatMessage.ImageInput(imageBytes)
+                        ]
+                    },
+                ]
+            };
+
+            var response = await openAI.Chat.CreateChatCompletionAsync(chatRequest);
+
+            var responseText = response?.Choices?.FirstOrDefault()?.Message.TextContent;
+            if (bool.Parse(responseText))
+            {
+                return new PackageResponse(true);
+            }
+        }
+
+        return new PackageResponse(false);
+    }
+}

HomeAutomation.Web/Data/PackageOptions.cs

@@ -0,0 +1,7 @@
+namespace HomeAutomation.Web.Data;
+
+public class PackageOptions
+{
+    public string OpenAIApiKey { get; set; }
+    public List<string> ImageUrls { get; set; }
+}

HomeAutomation.Web/Data/PackageResponse.cs

@@ -0,0 +1,11 @@
+namespace HomeAutomation.Web.Data;
+
+public class PackageResponse
+{
+    public PackageResponse(bool package)
+    {
+        Package = package;
+    }
+
+    public bool Package { get; set; }
+}

HomeAutomation.Web/HomeAutomation.Web.csproj

@@ -11,6 +11,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.6" />
     <PackageReference Include="MQTTnet" Version="4.3.3.952" />
+    <PackageReference Include="OpenAI" Version="1.11.0" />
     <PackageReference Include="System.Net.Http.Json" Version="8.0.0" />
   </ItemGroup>
 

HomeAutomation.Web/Program.cs

@@ -24,6 +24,7 @@
 builder.Services.Configure<ComputerOptions>(builder.Configuration.GetSection("Computer"));
 builder.Services.Configure<TeslaOptions>(builder.Configuration.GetSection("Tesla"));
 builder.Services.Configure<PushoverOptions>(builder.Configuration.GetSection("Pushover"));
+builder.Services.Configure<PackageOptions>(builder.Configuration.GetSection("Package"));
 builder.Services.AddMemoryCache();
 
 var app = builder.Build();