Test

Content

"; + var bytes = Encoding.UTF8.GetBytes(html); + using var stream = new MemoryStream(bytes); + var streamInfo = new StreamInfo(mimeType: "text/html"); + + // Act + var result = await converter.ConvertAsync(stream, streamInfo); + + // Assert + Assert.Contains("# Test", result.Markdown); + Assert.Contains("Content", result.Markdown); +} +``` + +## Converter Implementation Guidelines + +### Creating New Converters + +1. **Inherit from Base**: Consider if a base converter class would help +2. **Implement Interface**: All converters must implement `IDocumentConverter` +3. **Priority Assignment**: Lower numbers = higher priority (HTML = 100, Plain Text = 1000) +4. **Format Detection**: Be specific in `AcceptsInput` - check MIME type AND extension +5. **Error Handling**: Wrap third-party exceptions in `MarkItDownException` + +### Standard Converter Structure + +```csharp +public class YourFormatConverter : IDocumentConverter +{ + public int Priority => 200; // Between HTML(100) and PlainText(1000) + + public bool AcceptsInput(StreamInfo streamInfo) + { + return streamInfo.MimeType?.StartsWith("application/your-format") == true || + streamInfo.Extension?.ToLowerInvariant() == ".your-ext"; + } + + public async Task ConvertAsync( + Stream stream, + StreamInfo streamInfo, + CancellationToken cancellationToken = default) + { + try + { + // Reset stream position + if (stream.CanSeek) + stream.Position = 0; + + // Your conversion logic here + var markdown = await ConvertToMarkdownAsync(stream, cancellationToken); + + return new DocumentConverterResult( + markdown: markdown, + title: ExtractTitle(markdown) // Optional + ); + } + catch (Exception ex) when (!(ex is MarkItDownException)) + { + throw new MarkItDownException($"Failed to convert {streamInfo.Extension} file", ex); + } + } +} +``` + +## Package Management and Dependencies + +### NuGet Package References + +- **Core Dependencies**: Keep minimal - only what's absolutely needed +- **Version Pinning**: Use specific versions for reproducible builds +- **License Compatibility**: Ensure all dependencies are MIT-compatible +- **Security**: Regularly update packages for security fixes + +### Current Key Dependencies + +```xml + + + +``` + +## Testing Philosophy + +### Test Coverage Requirements + +- **Every Public Method**: Must have at least basic functionality tests +- **Error Conditions**: Test exception scenarios and edge cases +- **Integration Tests**: Test the full MarkItDown workflow +- **Format-Specific Tests**: Each converter needs comprehensive tests + +### Test Data Strategy + +```csharp +// Use test data that mirrors the original Python test vectors +public static class TestVectors +{ + public static readonly FileTestVector[] GeneralTestVectors = { + new FileTestVector( + filename: "test.html", + mimeType: "text/html", + mustInclude: new[] { "# Header", "**bold text**" }, + mustNotInclude: new[] { "", "

Run

Linked

Visit Example

Visible

Shown

Visible

Item 1
Item 2

Header 1	Header 2
Cell 1	Cell 2

This is a quote

This is inline code.

This is a code block

"; + var bytes = Encoding.UTF8.GetBytes(html); + using var stream = new MemoryStream(bytes); + var streamInfo = new StreamInfo(charset: Encoding.UTF8); + + // Act + var result = await converter.ConvertAsync(stream, streamInfo); + + // Assert + Assert.Contains("`inline code`", result.Markdown); + Assert.Contains("```", result.Markdown); + Assert.Contains("This is a code block", result.Markdown); + } + + [Fact] + public async Task ConvertAsync_EmptyHtml_ReturnsEmptyString() + { + // Arrange + var converter = new HtmlConverter(); + using var stream = new MemoryStream(); + var streamInfo = new StreamInfo(charset: Encoding.UTF8); + + // Act + var result = await converter.ConvertAsync(stream, streamInfo); + + // Assert + Assert.Equal(string.Empty, result.Markdown); + } +} diff --git a/tests/MarkItDown.Tests/Intelligence/AudioConverterIntelligenceTests.cs b/tests/MarkItDown.Tests/Intelligence/AudioConverterIntelligenceTests.cs new file mode 100644 index 000000000..fd552054f --- /dev/null +++ b/tests/MarkItDown.Tests/Intelligence/AudioConverterIntelligenceTests.cs @@ -0,0 +1,131 @@ +using MarkItDown; +using MarkItDown.Converters; +using MarkItDown.Intelligence; +using MarkItDown.Intelligence.Models; +using Shouldly; + +namespace MarkItDown.Tests.Intelligence; + +public class AudioConverterIntelligenceTests +{ + private sealed class StubMetadataExtractor(IReadOnlyDictionary metadata) : AudioConverter.IAudioMetadataExtractor + { + public Task> ExtractAsync(byte[] audioBytes, StreamInfo streamInfo, CancellationToken cancellationToken) + { + return Task.FromResult(metadata); + } + } + + private sealed class StubAudioTranscriber : AudioConverter.IAudioTranscriber + { + private readonly string? value; + + public StubAudioTranscriber(string? value = null) + { + this.value = value; + } + + public Task TranscribeAsync(byte[] audioBytes, StreamInfo streamInfo, CancellationToken cancellationToken) + { + return Task.FromResult(value); + } + } + + private sealed class StubMediaProvider(MediaTranscriptionResult? result) : IMediaTranscriptionProvider + { + public Task TranscribeAsync(Stream stream, StreamInfo streamInfo, MediaTranscriptionRequest? request = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(result); + } + } + + [Fact] + public async Task ConvertAsync_UsesMediaProviderSegmentsWhenAvailable() + { + var metadata = new Dictionary + { + ["Title"] = "Interview", + ["Duration"] = "00:02:00" + }; + + var segments = new List + { + new MediaTranscriptSegment("Hello", TimeSpan.Zero, TimeSpan.FromSeconds(30)), + new MediaTranscriptSegment("World", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60)) + }; + + var providerResult = new MediaTranscriptionResult(segments, language: "en-US"); + + var converter = new AudioConverter( + metadataExtractor: new StubMetadataExtractor(metadata), + transcriber: new StubAudioTranscriber(), + segmentOptions: SegmentOptions.Default, + mediaProvider: new StubMediaProvider(providerResult)); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "audio/wav", extension: ".wav", fileName: "sample.wav"); + + var result = await converter.ConvertAsync(stream, streamInfo); + + result.Markdown.ShouldContain("Audio Transcript"); + result.Segments.ShouldContain(s => s.Type == SegmentType.Audio && s.Markdown.Contains("Hello")); + result.Segments.ShouldContain(s => s.Type == SegmentType.Audio && s.Markdown.Contains("World")); + result.Segments.OfType().Count(s => s.Type == SegmentType.Metadata).ShouldBe(1); + } + + [Fact] + public async Task ConvertAsync_VideoInput_UsesProviderTranscriptAndVideoHeading() + { + var metadata = new Dictionary(); + var providerSegments = new List + { + new MediaTranscriptSegment( + "Provider transcript text", + TimeSpan.Zero, + TimeSpan.FromSeconds(5), + new Dictionary + { + [MetadataKeys.Speaker] = "Speaker #1", + [MetadataKeys.Sentiment] = "Neutral", + [MetadataKeys.SentimentScore] = "0.5", + [MetadataKeys.Topics] = "Health", + [MetadataKeys.Keywords] = "AGI", + [MetadataKeys.Confidence] = "0.72" + }) + }; + + var providerResult = new MediaTranscriptionResult( + providerSegments, + language: "en-US", + metadata: new Dictionary + { + [MetadataKeys.Provider] = MetadataValues.ProviderAzureVideoIndexer, + [MetadataKeys.Language] = "en-US", + [MetadataKeys.Duration] = "0:00:05", + [MetadataKeys.Speakers] = "Speaker #1", + [MetadataKeys.Sentiments] = "Neutral (100%)", + [MetadataKeys.Topics] = "Health (1x, 1.00)", + [MetadataKeys.Keywords] = "AGI (1x, 0.95)" + }); + + var converter = new AudioConverter( + metadataExtractor: new StubMetadataExtractor(metadata), + transcriber: new StubAudioTranscriber("Local transcript text"), + segmentOptions: SegmentOptions.Default, + mediaProvider: new StubMediaProvider(providerResult)); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "video/mp4", extension: ".mp4", fileName: "sample.mp4"); + + var result = await converter.ConvertAsync(stream, streamInfo); + + result.Markdown.ShouldContain("### Video Transcript"); + result.Markdown.ShouldContain("[00:00-00:05] Speaker #1"); + result.Markdown.ShouldContain("Emotion/Sentiment: Neutral"); + result.Markdown.ShouldContain("Topics: Health"); + result.Markdown.ShouldContain("Provider transcript text"); + result.Markdown.ShouldNotContain("Local transcript text"); + result.Markdown.ShouldContain("### Video Analysis"); + result.Markdown.ShouldContain("#### Topics"); + } +} diff --git a/tests/MarkItDown.Tests/Intelligence/AwsProvidersTests.cs b/tests/MarkItDown.Tests/Intelligence/AwsProvidersTests.cs new file mode 100644 index 000000000..048ed99db --- /dev/null +++ b/tests/MarkItDown.Tests/Intelligence/AwsProvidersTests.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Amazon.Rekognition.Model; +using Amazon.S3.Model; +using Amazon.Textract.Model; +using Amazon.TranscribeService.Model; +using MarkItDown; +using MarkItDown.Intelligence.Models; +using MarkItDown.Intelligence.Providers.Aws; +using Moq; +using Shouldly; +using Xunit; + +namespace MarkItDown.Tests.Intelligence; + +public class AwsProvidersTests +{ + [Fact] + public async Task DocumentProvider_ExtractsTableCells() + { + var blocks = new List + { + new() { Id = "page1", BlockType = "PAGE", Page = 1 }, + new() { Id = "line1", BlockType = "LINE", Page = 1, Text = "Name 42" }, + new() + { + Id = "table1", + BlockType = "TABLE", + Page = 1, + Relationships = new List + { + new Relationship { Type = "CHILD", Ids = new List { "cell1", "cell2" } } + } + }, + new() + { + Id = "cell1", + BlockType = "CELL", + Page = 1, + RowIndex = 1, + ColumnIndex = 1, + Relationships = new List + { + new Relationship { Type = "CHILD", Ids = new List { "word1" } } + } + }, + new() + { + Id = "cell2", + BlockType = "CELL", + Page = 1, + RowIndex = 1, + ColumnIndex = 2, + Relationships = new List + { + new Relationship { Type = "CHILD", Ids = new List { "word2" } } + } + }, + new() { Id = "word1", BlockType = "WORD", Page = 1, Text = "Name" }, + new() { Id = "word2", BlockType = "WORD", Page = 1, Text = "42" } + }; + + var response = new AnalyzeDocumentResponse + { + Blocks = blocks + }; + + var mockTextract = new Mock(); + mockTextract + .Setup(c => c.AnalyzeDocumentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(response); + + var provider = new AwsDocumentIntelligenceProvider(new AwsDocumentIntelligenceOptions + { + Region = "us-east-1" + }, mockTextract.Object); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "application/pdf", extension: ".pdf", fileName: "sample.pdf"); + + var result = await provider.AnalyzeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Tables.Count.ShouldBe(1); + result.Tables[0].Rows[0][0].ShouldBe("Name"); + result.Pages[0].Text.ShouldContain("Name 42"); + } + + [Fact] + public async Task ImageProvider_ProducesCaption() + { + var labelsResponse = new DetectLabelsResponse + { + Labels = new List + { + new Label + { + Name = "Document", + Confidence = 95f, + Instances = new List { new Instance() } + }, + new Label { Name = "Paper", Confidence = 85f } + } + }; + + var textResponse = new DetectTextResponse + { + TextDetections = new List + { + new TextDetection { Type = "LINE", DetectedText = "Invoice" }, + new TextDetection { Type = "LINE", DetectedText = "Total" } + } + }; + + var mockRekognition = new Mock(); + mockRekognition + .Setup(r => r.DetectLabelsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(labelsResponse); + mockRekognition + .Setup(r => r.DetectTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(textResponse); + + var provider = new AwsImageUnderstandingProvider(new AwsVisionOptions + { + Region = "us-east-1", + MinConfidence = 70f + }, mockRekognition.Object); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "image/png", extension: ".png", fileName: "image.png"); + + var result = await provider.AnalyzeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Caption.ShouldBe("Document"); + result.Text.ShouldNotBeNull(); + result.Text!.ShouldContain("Invoice"); + result.DetectedObjects.ShouldContain("Document"); + } + private static readonly string[] collection = new[] { "IN_PROGRESS", "COMPLETED" }; + + [Fact] + public async Task MediaProvider_ParsesTranscriptJson() + { + var transcriptJson = "{" + + "\"results\":{" + + "\"transcripts\":[{\"transcript\":\"Hello world.\"}]," + + "\"items\":[" + + "{\"type\":\"pronunciation\",\"start_time\":\"0.0\",\"end_time\":\"1.0\",\"alternatives\":[{\"content\":\"Hello\"}]}," + + "{\"type\":\"pronunciation\",\"start_time\":\"1.0\",\"end_time\":\"2.0\",\"alternatives\":[{\"content\":\"world\"}]}," + + "{\"type\":\"punctuation\",\"alternatives\":[{\"content\":\".\"}]}" + + "]}}"; + + var handler = new StubHttpMessageHandler(transcriptJson); + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/") + }; + + var mockS3 = new Mock(); + mockS3 + .Setup(s => s.PutObjectAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new PutObjectResponse()); + mockS3 + .Setup(s => s.DeleteObjectAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DeleteObjectResponse()); + + var statusQueue = new Queue(collection); + + var mockTranscribe = new Mock(); + mockTranscribe + .Setup(t => t.StartTranscriptionJobAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new StartTranscriptionJobResponse + { + TranscriptionJob = new TranscriptionJob + { + TranscriptionJobStatus = "IN_PROGRESS" + } + }); + + mockTranscribe + .Setup(t => t.GetTranscriptionJobAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => + { + var status = statusQueue.Count > 0 ? statusQueue.Dequeue() : "COMPLETED"; + return new GetTranscriptionJobResponse + { + TranscriptionJob = new TranscriptionJob + { + TranscriptionJobStatus = status, + Transcript = string.Equals(status, "COMPLETED", StringComparison.OrdinalIgnoreCase) + ? new Transcript { TranscriptFileUri = "https://example.com/transcript.json" } + : null + } + }; + }); + + var provider = new AwsMediaTranscriptionProvider(new AwsMediaIntelligenceOptions + { + Region = "us-east-1", + InputBucketName = "markitdown-input", + OutputBucketName = "markitdown-output", + DeleteInputOnCompletion = true, + PollingInterval = TimeSpan.FromMilliseconds(10) + }, mockTranscribe.Object, mockS3.Object, httpClient); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "audio/wav", extension: ".wav", fileName: "audio.wav"); + + var result = await provider.TranscribeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Segments.Count.ShouldBe(1); + result.Segments[0].Text.ShouldBe("Hello world."); + result.Segments[0].Start.ShouldBe(TimeSpan.Zero); + result.Segments[0].End.ShouldBe(TimeSpan.FromSeconds(2)); + + mockS3.Verify(s => s.PutObjectAsync(It.IsAny(), It.IsAny()), Times.Once); + mockS3.Verify(s => s.DeleteObjectAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly string content; + + public StubHttpMessageHandler(string content) + { + this.content = content; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(content) + }; + + return Task.FromResult(response); + } + } +} diff --git a/tests/MarkItDown.Tests/Intelligence/AzureMediaTranscriptionProviderTests.cs b/tests/MarkItDown.Tests/Intelligence/AzureMediaTranscriptionProviderTests.cs new file mode 100644 index 000000000..00c0a89ff --- /dev/null +++ b/tests/MarkItDown.Tests/Intelligence/AzureMediaTranscriptionProviderTests.cs @@ -0,0 +1,468 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using ManagedCode.Storage.FileSystem; +using ManagedCode.Storage.FileSystem.Options; +using MarkItDown; +using MarkItDown.Intelligence; +using MarkItDown.Intelligence.Models; +using MarkItDown.Intelligence.Providers.Azure; +using Shouldly; + +namespace MarkItDown.Tests.Intelligence; + +public class AzureMediaTranscriptionProviderTests +{ + private sealed class StubArmTokenService : ArmTokenService + { + private readonly string? token; + + public StubArmTokenService(string? token) + : base(token) + { + this.token = token; + } + + public override Task GetArmTokenAsync(CancellationToken cancellationToken) + { + return Task.FromResult(token); + } + } + + private sealed class SequenceHandler : HttpMessageHandler + { + private readonly Queue> responses = new(); + + public void Enqueue(Func factory) + { + responses.Enqueue(factory); + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (responses.Count == 0) + { + throw new InvalidOperationException("Unexpected request: " + request.RequestUri); + } + + var response = responses.Dequeue()(request); + return Task.FromResult(response); + } + } + + [Fact] + public async Task TranscribeAsync_ReturnsSegmentsFromVideoIndexer() + { + var handler = new SequenceHandler(); + + handler.Enqueue(request => + { + request.Method.ShouldBe(HttpMethod.Post); + request.RequestUri!.Host.ShouldBe("management.azure.com"); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"accessToken\": \"token123\", \"expirationTime\": \"2025-01-01T00:00:00Z\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(request => + { + request.Method.ShouldBe(HttpMethod.Post); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"id\": \"video123\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(request => + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"state\": \"Processed\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(request => + { + var transcriptJson = "{" + + "\"state\":\"Processed\"," + + "\"videos\":[{" + + "\"id\":\"index-123\"," + + "\"processingProgress\":\"100%\"," + + "\"insights\":{" + + "\"language\":\"en-US\"," + + "\"duration\":\"0:00:10\"," + + "\"speakers\":[{\"id\":1,\"name\":\"Speaker #1\"}]," + + "\"sentiments\":[{\"sentimentType\":\"Neutral\",\"averageScore\":0.5,\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:10\"}]}]," + + "\"topics\":[{\"name\":\"Health\",\"confidence\":0.998,\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:10\"}]}]," + + "\"keywords\":[{\"text\":\"AGI\",\"confidence\":0.95,\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:10\"}]}]," + + "\"transcript\":[" + + "{\"text\":\"Hello world\",\"confidence\":0.72,\"speakerId\":1,\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:05\"}]}," + + "{\"text\":\"Segment two\",\"confidence\":0.68,\"speakerId\":1,\"instances\":[{\"start\":\"0:00:05\",\"end\":\"0:00:10\"}]}" + + "]}}]}"; + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(transcriptJson, Encoding.UTF8, "application/json") + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }; + + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + SubscriptionId = "sub", + ResourceGroup = "rg", + AccountName = "account" + }; + + var provider = new AzureMediaTranscriptionProvider(options, httpClient, new StubArmTokenService("arm-token")); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "audio/wav", extension: ".wav", fileName: "sample.wav"); + + var result = await provider.TranscribeAsync(stream, streamInfo); + + result.ShouldNotBeNull(); + result!.Segments.Count.ShouldBe(2); + result.Segments[0].Text.ShouldBe("Hello world"); + result.Segments[0].Start.ShouldBe(TimeSpan.Zero); + result.Segments[0].End.ShouldBe(TimeSpan.FromSeconds(5)); + result.Segments[0].Metadata[MetadataKeys.Speaker].ShouldBe("Speaker #1"); + result.Segments[0].Metadata[MetadataKeys.Sentiment].ShouldBe("Neutral"); + result.Segments[0].Metadata[MetadataKeys.Topics].ShouldContain("Health"); + result.Segments[0].Metadata[MetadataKeys.Keywords].ShouldContain("AGI"); + result.Segments[1].Text.ShouldBe("Segment two"); + result.Metadata[MetadataKeys.Provider].ShouldBe(MetadataValues.ProviderAzureVideoIndexer); + result.Metadata[MetadataKeys.Speakers].ShouldContain("Speaker #1"); + result.Metadata[MetadataKeys.Sentiments].ShouldContain("Neutral"); + result.Metadata[MetadataKeys.VideoIndexerState].ShouldBe("Processed"); + result.Metadata[MetadataKeys.VideoIndexerIndexId].ShouldBe("index-123"); + result.Metadata[MetadataKeys.VideoIndexerProgress].ShouldBe("100%"); + result.Segments[0].Metadata[MetadataKeys.VideoIndexerState].ShouldBe("Processed"); + } + + [Fact] + public async Task TranscribeAsync_WhenRequestProvidesSourceUrl_UsesVideoUrlUpload() + { + var handler = new SequenceHandler(); + + handler.Enqueue(request => + { + request.Method.ShouldBe(HttpMethod.Post); + request.RequestUri!.Host.ShouldBe("management.azure.com"); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"accessToken\": \"token123\", \"expirationTime\": \"2025-01-01T00:00:00Z\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(request => + { + request.Method.ShouldBe(HttpMethod.Post); + request.RequestUri!.ToString().ShouldContain("/trial/Accounts/account/Videos?"); + request.RequestUri!.Query.ShouldContain("videoUrl=https%3A%2F%2Fstorage.example.com%2Fuploads%2Fclip.mp4%3Fsv%3D1%26sig%3Dabc"); + request.Content.ShouldBeNull(); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"id\": \"video123\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(_ => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"state\": \"Processed\" }", Encoding.UTF8, "application/json") + }); + + handler.Enqueue(_ => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{ \"state\":\"Processed\",\"videos\":[{\"insights\":{\"language\":\"en-US\",\"transcript\":[{\"text\":\"Hello world\",\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:02\"}]}]}}]}", + Encoding.UTF8, + "application/json") + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }; + + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + SubscriptionId = "sub", + ResourceGroup = "rg", + AccountName = "account" + }; + + var provider = new AzureMediaTranscriptionProvider(options, httpClient, new StubArmTokenService("arm-token")); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo( + mimeType: "video/mp4", + extension: ".mp4", + fileName: "sample.mp4", + url: "https://platform.example.com/watch/abc123"); + var request = new MediaTranscriptionRequest( + PreferredProvider: MediaTranscriptionProviderKind.Azure, + Language: "English", + SourceUrl: "https://storage.example.com/uploads/clip.mp4?sv=1&sig=abc"); + + var result = await provider.TranscribeAsync(stream, streamInfo, request); + + result.ShouldNotBeNull(); + result!.Segments.Count.ShouldBe(1); + result.Segments[0].Text.ShouldBe("Hello world"); + } + + [Fact] + public async Task TranscribeAsync_WhenRequestSourceUrlIsNotHttp_ThrowsFileConversionException() + { + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + AccountName = "account", + SubscriptionId = "sub", + ResourceGroup = "rg" + }; + + var provider = new AzureMediaTranscriptionProvider(options, new HttpClient(new SequenceHandler()) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }, new StubArmTokenService("arm-token")); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "video/mp4", extension: ".mp4", fileName: "sample.mp4"); + var request = new MediaTranscriptionRequest( + PreferredProvider: MediaTranscriptionProviderKind.Azure, + Language: "English", + SourceUrl: "ftp://storage.example.com/uploads/clip.mp4"); + + var exception = await Should.ThrowAsync( + () => provider.TranscribeAsync(stream, streamInfo, request)); + + exception.Message.ShouldContain("absolute http/https URL"); + } + + [Fact] + public async Task TranscribeAsync_WhenUploadRouteStream_UsesMultipartUploadEvenWhenUrlsExist() + { + var handler = new SequenceHandler(); + + handler.Enqueue(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"accessToken\": \"token123\", \"expirationTime\": \"2025-01-01T00:00:00Z\" }", Encoding.UTF8, "application/json") + }); + + handler.Enqueue(request => + { + request.Method.ShouldBe(HttpMethod.Post); + request.RequestUri!.ToString().ShouldContain("/trial/Accounts/account/Videos?"); + request.RequestUri!.Query.ShouldNotContain("videoUrl="); + request.Content.ShouldNotBeNull(); + request.Content.ShouldBeOfType(); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"id\": \"video123\" }", Encoding.UTF8, "application/json") + }; + }); + + handler.Enqueue(_ => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"state\": \"Processed\" }", Encoding.UTF8, "application/json") + }); + + handler.Enqueue(_ => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{ \"state\":\"Processed\",\"videos\":[{\"insights\":{\"language\":\"en-US\",\"transcript\":[{\"text\":\"Hello world\",\"instances\":[{\"start\":\"0:00:00\",\"end\":\"0:00:02\"}]}]}}]}", + Encoding.UTF8, + "application/json") + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }; + + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + SubscriptionId = "sub", + ResourceGroup = "rg", + AccountName = "account" + }; + + var provider = new AzureMediaTranscriptionProvider(options, httpClient, new StubArmTokenService("arm-token")); + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo( + mimeType: "video/mp4", + extension: ".mp4", + fileName: "sample.mp4", + url: "https://platform.example.com/watch/abc123"); + var request = new MediaTranscriptionRequest( + PreferredProvider: MediaTranscriptionProviderKind.Azure, + Language: "English", + SourceUrl: "https://storage.example.com/uploads/clip.mp4?sv=1&sig=abc", + UploadRoute: MediaUploadRoute.Stream); + + var result = await provider.TranscribeAsync(stream, streamInfo, request); + + result.ShouldNotBeNull(); + result!.Segments.Count.ShouldBe(1); + } + + [Fact] + public async Task TranscribeAsync_WhenUploadRouteStorageUrlWithFileSystemStorage_UploadsFileAndFailsForNonHttpUri() + { + var handler = new SequenceHandler(); + + handler.Enqueue(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{ \"accessToken\": \"token123\", \"expirationTime\": \"2025-01-01T00:00:00Z\" }", Encoding.UTF8, "application/json") + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }; + + var storageRoot = Path.Combine(Path.GetTempPath(), $"markitdown-storage-{Guid.NewGuid():N}"); + Directory.CreateDirectory(storageRoot); + + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + SubscriptionId = "sub", + ResourceGroup = "rg", + AccountName = "account", + UploadStorageFactory = () => new FileSystemStorage(new FileSystemStorageOptions + { + BaseFolder = storageRoot, + CreateContainerIfNotExists = true + }), + UploadStorageDirectoryResolver = _ => "uploads" + }; + + var provider = new AzureMediaTranscriptionProvider(options, httpClient, new StubArmTokenService("arm-token")); + var path = Path.Combine(Path.GetTempPath(), $"markitdown-video-{Guid.NewGuid():N}.mp4"); + await File.WriteAllBytesAsync(path, [1, 2, 3, 4]); + + try + { + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo( + mimeType: "video/mp4", + extension: ".mp4", + fileName: "sample.mp4", + localPath: path); + var request = new MediaTranscriptionRequest( + PreferredProvider: MediaTranscriptionProviderKind.Azure, + Language: "English", + UploadRoute: MediaUploadRoute.StorageUrl); + + var exception = await Should.ThrowAsync( + () => provider.TranscribeAsync(stream, streamInfo, request)); + + exception.Message.ShouldContain("Storage upload result URI must be an absolute http/https URL"); + + var uploadedFiles = Directory.GetFiles(storageRoot, "*.mp4", SearchOption.AllDirectories); + uploadedFiles.Length.ShouldBe(1); + var uploadedFileInfo = new FileInfo(uploadedFiles[0]); + uploadedFileInfo.Exists.ShouldBeTrue(); + uploadedFileInfo.Length.ShouldBe(4); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + + if (Directory.Exists(storageRoot)) + { + Directory.Delete(storageRoot, recursive: true); + } + } + } + + [Fact] + public async Task TranscribeAsync_WhenUploadRouteStorageUrlWithoutProvider_ThrowsFileConversionException() + { + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + AccountName = "account", + SubscriptionId = "sub", + ResourceGroup = "rg" + }; + + var provider = new AzureMediaTranscriptionProvider(options, new HttpClient(new SequenceHandler()) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }, new StubArmTokenService("arm-token")); + + var path = Path.Combine(Path.GetTempPath(), $"markitdown-video-{Guid.NewGuid():N}.mp4"); + await File.WriteAllBytesAsync(path, [1, 2, 3, 4]); + + try + { + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "video/mp4", extension: ".mp4", fileName: "sample.mp4", localPath: path); + var request = new MediaTranscriptionRequest( + PreferredProvider: MediaTranscriptionProviderKind.Azure, + Language: "English", + UploadRoute: MediaUploadRoute.StorageUrl); + + var exception = await Should.ThrowAsync( + () => provider.TranscribeAsync(stream, streamInfo, request)); + + exception.Message.ShouldContain("UploadStorageFactory"); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } + + [Fact] + public async Task TranscribeAsync_ReturnsNullWhenTokenUnavailable() + { + var options = new AzureMediaIntelligenceOptions + { + AccountId = "account", + Location = "trial", + AccountName = "account", + SubscriptionId = "sub", + ResourceGroup = "rg" + }; + + var provider = new AzureMediaTranscriptionProvider(options, new HttpClient(new SequenceHandler()) + { + BaseAddress = new Uri("https://api.videoindexer.ai/") + }, new StubArmTokenService(null)); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "audio/wav", extension: ".wav"); + + var result = await provider.TranscribeAsync(stream, streamInfo); + result.ShouldBeNull(); + } +} diff --git a/tests/MarkItDown.Tests/Intelligence/GoogleProvidersTests.cs b/tests/MarkItDown.Tests/Intelligence/GoogleProvidersTests.cs new file mode 100644 index 000000000..543868b39 --- /dev/null +++ b/tests/MarkItDown.Tests/Intelligence/GoogleProvidersTests.cs @@ -0,0 +1,238 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Google.Api.Gax.Grpc; +using Google.Cloud.DocumentAI.V1; +using Google.Cloud.Speech.V1; +using Google.Cloud.Vision.V1; +using Google.Protobuf.WellKnownTypes; +using MarkItDown; +using MarkItDown.Intelligence.Models; +using MarkItDown.Intelligence.Providers.Google; +using Moq; +using Shouldly; +using Xunit; + +namespace MarkItDown.Tests.Intelligence; + +public class GoogleProvidersTests +{ + [Fact] + public async Task DocumentProvider_ConvertsPagesAndTables() + { + var documentText = "Header Value Row 42"; + var document = new Document + { + Text = documentText + }; + + var pageAnchor = new Document.Types.TextAnchor(); + pageAnchor.TextSegments.Add(new Document.Types.TextAnchor.Types.TextSegment + { + StartIndex = 0, + EndIndex = documentText.Length + }); + + var headerAnchor = new Document.Types.TextAnchor(); + headerAnchor.TextSegments.Add(new Document.Types.TextAnchor.Types.TextSegment + { + StartIndex = 0, + EndIndex = 6 + }); + + var valueAnchor = new Document.Types.TextAnchor(); + valueAnchor.TextSegments.Add(new Document.Types.TextAnchor.Types.TextSegment + { + StartIndex = 7, + EndIndex = 12 + }); + + var bodyHeaderAnchor = new Document.Types.TextAnchor(); + bodyHeaderAnchor.TextSegments.Add(new Document.Types.TextAnchor.Types.TextSegment + { + StartIndex = 13, + EndIndex = 16 + }); + + var bodyValueAnchor = new Document.Types.TextAnchor(); + bodyValueAnchor.TextSegments.Add(new Document.Types.TextAnchor.Types.TextSegment + { + StartIndex = 17, + EndIndex = documentText.Length + }); + + var table = new Document.Types.Page.Types.Table(); + table.HeaderRows.Add(new Document.Types.Page.Types.Table.Types.TableRow + { + Cells = + { + new Document.Types.Page.Types.Table.Types.TableCell + { + Layout = new Document.Types.Page.Types.Layout { TextAnchor = headerAnchor } + }, + new Document.Types.Page.Types.Table.Types.TableCell + { + Layout = new Document.Types.Page.Types.Layout { TextAnchor = valueAnchor } + } + } + }); + + table.BodyRows.Add(new Document.Types.Page.Types.Table.Types.TableRow + { + Cells = + { + new Document.Types.Page.Types.Table.Types.TableCell + { + Layout = new Document.Types.Page.Types.Layout { TextAnchor = bodyHeaderAnchor } + }, + new Document.Types.Page.Types.Table.Types.TableCell + { + Layout = new Document.Types.Page.Types.Layout { TextAnchor = bodyValueAnchor } + } + } + }); + + var page = new Document.Types.Page + { + PageNumber = 1, + Layout = new Document.Types.Page.Types.Layout { TextAnchor = pageAnchor }, + Dimension = new Document.Types.Page.Types.Dimension + { + Width = 8.5f, + Height = 11f, + Unit = "inch" + }, + Tables = { table } + }; + + document.Pages.Add(page); + + var mockClient = new Mock(); + mockClient + .Setup(c => c.ProcessDocumentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ProcessResponse { Document = document }); + + var provider = new GoogleDocumentIntelligenceProvider(new GoogleDocumentIntelligenceOptions + { + ProjectId = "proj", + Location = "us", + ProcessorId = "processor" + }, mockClient.Object); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "application/pdf", extension: ".pdf", fileName: "sample.pdf"); + + var result = await provider.AnalyzeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Pages.Count.ShouldBe(1); + result.Pages[0].Text.ShouldContain("Header"); + result.Tables.Count.ShouldBe(1); + result.Tables[0].Rows.Count.ShouldBe(2); + result.Tables[0].Rows[1][1].ShouldBe("42"); + } + + [Fact] + public async Task ImageProvider_ProducesCaptionAndText() + { + var annotateResponse = new BatchAnnotateImagesResponse(); + annotateResponse.Responses.Add(new AnnotateImageResponse + { + LabelAnnotations = + { + new EntityAnnotation { Description = "Laptop", Score = 0.95f }, + new EntityAnnotation { Description = "Desk", Score = 0.80f } + }, + FullTextAnnotation = new TextAnnotation { Text = "Invoice" }, + LocalizedObjectAnnotations = + { + new LocalizedObjectAnnotation { Name = "Computer", Score = 0.88f } + }, + SafeSearchAnnotation = new SafeSearchAnnotation + { + Adult = Likelihood.Unlikely, + Medical = Likelihood.VeryUnlikely, + Racy = Likelihood.Unlikely, + Violence = Likelihood.VeryUnlikely + } + }); + + var mockClient = new Mock(); + mockClient + .Setup(c => c.BatchAnnotateImagesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(annotateResponse); + + var provider = new GoogleImageUnderstandingProvider(new GoogleVisionOptions + { + MaxLabels = 5 + }, mockClient.Object); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + var streamInfo = new StreamInfo(mimeType: "image/png", extension: ".png", fileName: "image.png"); + + var result = await provider.AnalyzeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Caption.ShouldBe("Laptop"); + result.Text.ShouldBe("Invoice"); + result.DetectedObjects.ShouldContain("Computer"); + result.Metadata["safeSearch.adult"].ShouldBe(Likelihood.Unlikely.ToString()); + } + + [Fact] + public async Task MediaProvider_ReturnsTimedSegments() + { + var recognizeResponse = new RecognizeResponse + { + Results = + { + new SpeechRecognitionResult + { + Alternatives = + { + new SpeechRecognitionAlternative + { + Transcript = "Hello world.", + Confidence = 0.9f, + Words = + { + new WordInfo + { + StartTime = Duration.FromTimeSpan(TimeSpan.Zero), + EndTime = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)) + }, + new WordInfo + { + StartTime = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)), + EndTime = Duration.FromTimeSpan(TimeSpan.FromSeconds(2)) + } + } + } + } + } + } + }; + + var mockClient = new Mock(); + mockClient + .Setup(c => c.RecognizeAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(recognizeResponse); + + var provider = new GoogleMediaTranscriptionProvider(new GoogleMediaIntelligenceOptions + { + LanguageCode = "en-US" + }, mockClient.Object); + + await using var stream = new MemoryStream(new byte[] { 1, 2, 3, 4 }); + var streamInfo = new StreamInfo(mimeType: "audio/wav", extension: ".wav", fileName: "audio.wav"); + + var result = await provider.TranscribeAsync(stream, streamInfo, request: null, CancellationToken.None); + + result.ShouldNotBeNull(); + result!.Segments.Count.ShouldBe(1); + result.Segments[0].Text.ShouldBe("Hello world."); + result.Segments[0].Start.ShouldBe(TimeSpan.Zero); + result.Segments[0].End.ShouldBe(TimeSpan.FromSeconds(2)); + } +} diff --git a/tests/MarkItDown.Tests/Intelligence/ImageChatEnricherTests.cs b/tests/MarkItDown.Tests/Intelligence/ImageChatEnricherTests.cs new file mode 100644 index 000000000..2e6e0400e --- /dev/null +++ b/tests/MarkItDown.Tests/Intelligence/ImageChatEnricherTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using MarkItDown; +using MarkItDown.Intelligence; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; + +namespace MarkItDown.Tests.Intelligence; + +public class ImageChatEnricherTests +{ + [Fact] + public async Task EnrichAsync_AttachesImageBinaryToChatMessage() + { + // Arrange + var image = new ImageArtifact(new byte[] { 0x1, 0x2, 0x3 }, "image/png", pageNumber: 7, source: "page-7.png"); + var streamInfo = new StreamInfo(fileName: "document.docx"); + var responsePayload = JsonSerializer.Serialize(new + { + description = "Dashboard overview with KPI cards.", + textRegions = new[] + { + new { label = "Heading", text = "Dashboard Overview" } + }, + diagrams = Array.Empty

Burn ID	Planned Δv (m/s)	Actual Δv (m/s)	Related Asset
MNV-2045-17	12.4	12.6	Telemetry Batch 3021
MNV-2045-18	4.8	4.9	Engineering Report 44-B