Add Android podcast background blur
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
parent
e72df234bc
commit
689b1a48c2
18 changed files with 23035 additions and 22 deletions
1
.gitattributes
vendored
1
.gitattributes
vendored
|
|
@ -1,5 +1,6 @@
|
|||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
*.onnx binary
|
||||
|
||||
# Custom for Visual Studio
|
||||
*.cs diff=csharp
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
### Added
|
||||
|
||||
* [PostIt.Android/Podcasts] Add an optional persisted background blur for camera preview and broadcasting. Run the bundled PPHumanSeg model locally with ONNX Runtime and composite with OpenGL before encoding; preserve the unfiltered pipeline for calls and OBS.
|
||||
|
||||
* [Podcasts/OBS/PostIt] Replace the MediaMTX relay with direct encrypted SRT ingestion managed by [FFmpeg.NET](https://github.com/cmxl/FFmpeg.NET). Reserve bounded UDP listeners on activation, display a secret SRT URL with an empty OBS stream-key field, start on received media, preserve revocation/expiry/quotas and authenticated replay, and invalidate in-memory activations after restart. Remove relay configuration and its authentication endpoint; update deployment, privacy and real-media regressions. Use separate request connections to the shared SQLite test database for reliable concurrent podcast requests.
|
||||
|
||||
* [Podcasts/OBS/PostIt] Add opt-in MediaMTX RTMPS ingestion for programmed emissions, activated explicitly from PostIt with one-time scoped, expiring and revocable keys. Start on publisher connection, remux H.264/AAC into the existing authenticated segments and MP4 replay workflow, bound concurrency/duration/buffering, enforce consumption and effective disk quotas, account storage with deletion refunds, and report owner-only ingest errors. Add relay configuration, installation instructions and real synthetic RTMP/RTMPS-to-replay regressions, including active revocation.
|
||||
|
|
|
|||
16
INSTALL.md
16
INSTALL.md
|
|
@ -526,6 +526,22 @@ at IDR frames, with timestamps used for segment durations. Closed segments
|
|||
enter the same ordered upload/retry/quota/replay pipeline as desktop.
|
||||
Devices without a supported camera/encoder report a startup error.
|
||||
|
||||
On Android, the optional **Flouter le decor de la camera** setting is saved
|
||||
locally and disabled by default. It applies to podcast camera preview and
|
||||
recording, not calls or OBS. The bundled Apache-2.0 PPHumanSeg model from
|
||||
[OpenCV Zoo](https://github.com/opencv/opencv_zoo/tree/main/models/human_segmentation_pphumanseg)
|
||||
is evaluated locally with ONNX Runtime; OpenGL blends the blurred background
|
||||
before the native encoder. No frames are sent to a third party. Segmentation
|
||||
is imperfect and can expose parts of the background: check the preview and do
|
||||
not rely on it to conceal sensitive surroundings. Performance and battery
|
||||
usage depend on the device. Capture stops on processing failure.
|
||||
The bundled `pphumanseg.onnx` is the OpenCV Zoo 2023mar model (SHA-256
|
||||
`552d8a984054e59b5d773d24b9b12022b22046ceb2bbc4c9aaeaceb36a9ddf24`);
|
||||
its Apache-2.0 license is included beside the asset.
|
||||
ONNX Runtime's Android library requires API 24; it is only loaded by the
|
||||
podcast capture code, which is restricted to API 26 and later. The manifest
|
||||
override preserves PostIt's existing API 23 minimum for its other features.
|
||||
|
||||
Camera frames are rotated through an OpenGL surface before encoding, so both
|
||||
live playback and replays have upright pixels without relying on rotation
|
||||
metadata (not supported by the MPEG-TS recorder). The sensor, front/rear lens
|
||||
|
|
|
|||
|
|
@ -39,5 +39,6 @@
|
|||
<PackageVersion Include="Xamarin.AndroidX.Lifecycle.Process" Version="2.11.0.1" />
|
||||
<PackageVersion Include="Xamarin.UITest" Version="4.4.2" />
|
||||
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.13" />
|
||||
<PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.31.1" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
|
|
|||
203
src/PostIt/PostIt.Android/Assets/PodcastBackground/LICENSE
Normal file
203
src/PostIt/PostIt.Android/Assets/PodcastBackground/LICENSE
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
22323
src/PostIt/PostIt.Android/Assets/PodcastBackground/pphumanseg.onnx
Normal file
22323
src/PostIt/PostIt.Android/Assets/PodcastBackground/pphumanseg.onnx
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -36,6 +36,8 @@ internal static class PlatformBootstrap
|
|||
Platform.RequestPodcastPermissionsAsync = AndroidPodcastCapture.RequestPermissionsAsync;
|
||||
Platform.CreatePodcastCaptureAsync = AndroidPodcastCapture.StartAsync;
|
||||
Platform.CreatePodcastPreviewAsync = AndroidPodcastCapture.StartPreviewAsync;
|
||||
Platform.CreateBlurredPodcastCaptureAsync = AndroidPodcastCapture.StartBlurredAsync;
|
||||
Platform.CreateBlurredPodcastPreviewAsync = AndroidPodcastCapture.StartBlurredPreviewAsync;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
<ItemGroup>
|
||||
<PackageReference Include="Avalonia.Android" />
|
||||
<PackageReference Include="LibVLCSharp" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxRuntime" />
|
||||
<PackageReference Include="VideoLAN.LibVLC.Android" />
|
||||
<PackageReference Include="Xamarin.AndroidX.Core.SplashScreen" />
|
||||
<PackageReference Include="Xamarin.AndroidX.Browser" />
|
||||
|
|
@ -37,6 +38,7 @@
|
|||
<Compile Include="..\PostIt.Tests\AndroidChatServiceTestActivity.cs" Link="Tests\AndroidChatServiceTestActivity.cs" />
|
||||
<Compile Include="..\PostIt.Tests\AndroidPodcastCaptureTestActivity.cs" Link="Tests\AndroidPodcastCaptureTestActivity.cs" />
|
||||
<Compile Include="..\PostIt.Tests\AndroidCameraFrameRotationTestActivity.cs" Link="Tests\AndroidCameraFrameRotationTestActivity.cs" />
|
||||
<Compile Include="..\PostIt.Tests\AndroidPodcastBlurTestActivity.cs" Link="Tests\AndroidPodcastBlurTestActivity.cs" />
|
||||
<Compile Include="..\PostIt.Tests\AndroidLanCallTestActivity.cs" Link="Tests\AndroidLanCallTestActivity.cs" />
|
||||
<Compile Include="..\PostIt.Tests\AndroidScreenAwakeTestActivity.cs" Link="Tests\AndroidScreenAwakeTestActivity.cs" />
|
||||
</ItemGroup>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android" android:installLocation="auto">
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools" android:installLocation="auto">
|
||||
<!-- ONNX Runtime is loaded only for podcast capture, which requires API 26. -->
|
||||
<uses-sdk tools:overrideLibrary="ai.onnxruntime" />
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Android.Graphics;
|
||||
using Android.Content;
|
||||
using Android.Opengl;
|
||||
using Android.OS;
|
||||
using Android.Views;
|
||||
|
|
@ -17,9 +18,32 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
attribute vec2 textureCoordinate;
|
||||
uniform mat4 textureMatrix;
|
||||
varying vec2 uv;
|
||||
varying vec2 maskUv;
|
||||
void main() {
|
||||
gl_Position = vec4(position, 0.0, 1.0);
|
||||
uv = (textureMatrix * vec4(textureCoordinate, 0.0, 1.0)).xy;
|
||||
maskUv = (position + vec2(1.0)) * 0.5;
|
||||
}
|
||||
""";
|
||||
private const string BlurFragmentShader = """
|
||||
#extension GL_OES_EGL_image_external : require
|
||||
precision mediump float;
|
||||
uniform samplerExternalOES cameraTexture;
|
||||
uniform sampler2D personMask;
|
||||
uniform vec2 blurStep;
|
||||
varying vec2 uv;
|
||||
varying vec2 maskUv;
|
||||
void main() {
|
||||
vec4 original = texture2D(cameraTexture, uv);
|
||||
vec4 blurred = vec4(0.0);
|
||||
for (int x = -2; x <= 2; x++) {
|
||||
for (int y = -2; y <= 2; y++) {
|
||||
blurred += texture2D(cameraTexture, uv + vec2(float(x), float(y)) * blurStep);
|
||||
}
|
||||
}
|
||||
blurred /= 25.0;
|
||||
float person = texture2D(personMask, maskUv).r;
|
||||
gl_FragColor = mix(blurred, original, person);
|
||||
}
|
||||
""";
|
||||
private const string FragmentShader = """
|
||||
|
|
@ -59,7 +83,17 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
private int _program;
|
||||
private int _textureId;
|
||||
private int _matrixLocation;
|
||||
private int _blurProgram;
|
||||
private int _blurMatrixLocation;
|
||||
private int _maskTexture;
|
||||
private int _samplingTexture;
|
||||
private int _framebuffer;
|
||||
private IPodcastPersonSegmenter? _segmenter;
|
||||
private ByteBuffer? _samplingBuffer;
|
||||
private ByteBuffer? _maskBuffer;
|
||||
private byte[]? _samplingBytes;
|
||||
private bool _rendering;
|
||||
private bool _blackout;
|
||||
private bool _disposed;
|
||||
private int _rotation;
|
||||
private int _remainingRotation = -1;
|
||||
|
|
@ -76,12 +110,13 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
}
|
||||
|
||||
internal static async Task<AndroidCameraFrameRotator> CreateAsync(
|
||||
Surface? encoderSurface, int width, int height, int rotation, Action<Exception> failed)
|
||||
Surface? encoderSurface, int width, int height, int rotation, Action<Exception> failed, Context? blurContext = null,
|
||||
Func<Context, IPodcastPersonSegmenter>? segmenterFactory = null)
|
||||
{
|
||||
var rotator = new AndroidCameraFrameRotator(failed);
|
||||
try
|
||||
{
|
||||
await rotator.InvokeAsync(() => rotator.Initialize(encoderSurface, width, height, rotation));
|
||||
await rotator.InvokeAsync(() => rotator.Initialize(encoderSurface, width, height, rotation, blurContext, segmenterFactory));
|
||||
return rotator;
|
||||
}
|
||||
catch
|
||||
|
|
@ -91,7 +126,8 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
}
|
||||
}
|
||||
|
||||
private void Initialize(Surface? encoderSurface, int width, int height, int rotation)
|
||||
private void Initialize(Surface? encoderSurface, int width, int height, int rotation, Context? blurContext,
|
||||
Func<Context, IPodcastPersonSegmenter>? segmenterFactory)
|
||||
{
|
||||
_rotation = rotation;
|
||||
_inputWidth = width;
|
||||
|
|
@ -126,7 +162,7 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
if (!EGL14.EglMakeCurrent(_display, _window, _window, _context))
|
||||
throw EglError("Activation du contexte EGL");
|
||||
|
||||
_program = CreateProgram();
|
||||
_program = CreateProgram(FragmentShader);
|
||||
var ids = new int[1];
|
||||
GLES20.GlGenTextures(1, ids, 0);
|
||||
_textureId = ids[0];
|
||||
|
|
@ -142,6 +178,7 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
BindAttribute("textureCoordinate", _coordinates);
|
||||
_matrixLocation = GLES20.GlGetUniformLocation(_program, "textureMatrix");
|
||||
GLES20.GlUniform1i(GLES20.GlGetUniformLocation(_program, "cameraTexture"), 0);
|
||||
if (blurContext is not null) InitializeBlur(blurContext, segmenterFactory);
|
||||
_outputWidth = rotation is 90 or 270 ? height : width;
|
||||
_outputHeight = rotation is 90 or 270 ? width : height;
|
||||
GLES20.GlViewport(0, 0, _outputWidth, _outputHeight);
|
||||
|
|
@ -181,6 +218,82 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
}
|
||||
}
|
||||
|
||||
private void InitializeBlur(Context context, Func<Context, IPodcastPersonSegmenter>? segmenterFactory)
|
||||
{
|
||||
_segmenter = segmenterFactory is null ? new PodcastPersonSegmenter(context) : segmenterFactory(context);
|
||||
_samplingBytes = new byte[PodcastPersonSegmenter.Size * PodcastPersonSegmenter.Size * 4];
|
||||
_samplingBuffer = ByteBuffer.AllocateDirect(_samplingBytes.Length)
|
||||
?? throw new InvalidOperationException("Tampon de segmentation indisponible.");
|
||||
_maskBuffer = ByteBuffer.AllocateDirect(PodcastPersonSegmenter.Size * PodcastPersonSegmenter.Size)
|
||||
?? throw new InvalidOperationException("Tampon du masque indisponible.");
|
||||
_blurProgram = CreateProgram(BlurFragmentShader);
|
||||
GLES20.GlUseProgram(_blurProgram);
|
||||
_blurMatrixLocation = GLES20.GlGetUniformLocation(_blurProgram, "textureMatrix");
|
||||
GLES20.GlUniform1i(GLES20.GlGetUniformLocation(_blurProgram, "cameraTexture"), 0);
|
||||
GLES20.GlUniform1i(GLES20.GlGetUniformLocation(_blurProgram, "personMask"), 1);
|
||||
GLES20.GlUniform2f(GLES20.GlGetUniformLocation(_blurProgram, "blurStep"), 4f / _inputWidth, 4f / _inputHeight);
|
||||
var ids = new int[1];
|
||||
GLES20.GlGenTextures(1, ids, 0);
|
||||
_maskTexture = ids[0];
|
||||
GLES20.GlActiveTexture(GLES20.GlTexture1);
|
||||
GLES20.GlBindTexture(GLES20.GlTexture2d, _maskTexture);
|
||||
SetTextureParameters();
|
||||
GLES20.GlTexImage2D(GLES20.GlTexture2d, 0, GLES20.GlLuminance,
|
||||
PodcastPersonSegmenter.Size, PodcastPersonSegmenter.Size, 0, GLES20.GlLuminance, GLES20.GlUnsignedByte, null);
|
||||
GLES20.GlGenTextures(1, ids, 0);
|
||||
_samplingTexture = ids[0];
|
||||
GLES20.GlBindTexture(GLES20.GlTexture2d, _samplingTexture);
|
||||
SetTextureParameters();
|
||||
GLES20.GlTexImage2D(GLES20.GlTexture2d, 0, GLES20.GlRgba,
|
||||
PodcastPersonSegmenter.Size, PodcastPersonSegmenter.Size, 0, GLES20.GlRgba, GLES20.GlUnsignedByte, null);
|
||||
GLES20.GlGenFramebuffers(1, ids, 0);
|
||||
_framebuffer = ids[0];
|
||||
GLES20.GlBindFramebuffer(GLES20.GlFramebuffer, _framebuffer);
|
||||
GLES20.GlFramebufferTexture2D(GLES20.GlFramebuffer, GLES20.GlColorAttachment0,
|
||||
GLES20.GlTexture2d, _samplingTexture, 0);
|
||||
if (GLES20.GlCheckFramebufferStatus(GLES20.GlFramebuffer) != GLES20.GlFramebufferComplete)
|
||||
throw new InvalidOperationException("Surface de segmentation OpenGL indisponible.");
|
||||
GLES20.GlBindFramebuffer(GLES20.GlFramebuffer, 0);
|
||||
GLES20.GlBindTexture(GLES20.GlTexture2d, _maskTexture);
|
||||
GLES20.GlActiveTexture(GLES20.GlTexture0);
|
||||
GLES20.GlUseProgram(_program);
|
||||
BindAttribute("position", _vertices!);
|
||||
BindAttribute("textureCoordinate", _coordinates!);
|
||||
CheckGl();
|
||||
}
|
||||
|
||||
private static void SetTextureParameters()
|
||||
{
|
||||
GLES20.GlTexParameteri(GLES20.GlTexture2d, GLES20.GlTextureMinFilter, GLES20.GlLinear);
|
||||
GLES20.GlTexParameteri(GLES20.GlTexture2d, GLES20.GlTextureMagFilter, GLES20.GlLinear);
|
||||
GLES20.GlTexParameteri(GLES20.GlTexture2d, GLES20.GlTextureWrapS, GLES20.GlClampToEdge);
|
||||
GLES20.GlTexParameteri(GLES20.GlTexture2d, GLES20.GlTextureWrapT, GLES20.GlClampToEdge);
|
||||
}
|
||||
|
||||
private void UpdatePersonMask()
|
||||
{
|
||||
GLES20.GlBindFramebuffer(GLES20.GlFramebuffer, _framebuffer);
|
||||
GLES20.GlViewport(0, 0, PodcastPersonSegmenter.Size, PodcastPersonSegmenter.Size);
|
||||
GLES20.GlDrawArrays(GLES20.GlTriangleStrip, 0, 4);
|
||||
_samplingBuffer!.Position(0);
|
||||
GLES20.GlReadPixels(0, 0, PodcastPersonSegmenter.Size, PodcastPersonSegmenter.Size,
|
||||
GLES20.GlRgba, GLES20.GlUnsignedByte, _samplingBuffer);
|
||||
CheckGl();
|
||||
_samplingBuffer.Position(0);
|
||||
_samplingBuffer.Get(_samplingBytes!);
|
||||
var mask = _segmenter!.Segment(_samplingBytes!);
|
||||
_maskBuffer!.Position(0);
|
||||
_maskBuffer.Put(mask);
|
||||
_maskBuffer.Position(0);
|
||||
GLES20.GlActiveTexture(GLES20.GlTexture1);
|
||||
GLES20.GlBindTexture(GLES20.GlTexture2d, _maskTexture);
|
||||
GLES20.GlTexSubImage2D(GLES20.GlTexture2d, 0, 0, 0, PodcastPersonSegmenter.Size,
|
||||
PodcastPersonSegmenter.Size, GLES20.GlLuminance, GLES20.GlUnsignedByte, _maskBuffer);
|
||||
GLES20.GlActiveTexture(GLES20.GlTexture0);
|
||||
GLES20.GlBindFramebuffer(GLES20.GlFramebuffer, 0);
|
||||
CheckGl();
|
||||
}
|
||||
|
||||
private void DrawFrame(SurfaceTexture texture)
|
||||
{
|
||||
if (!_rendering) return;
|
||||
|
|
@ -191,6 +304,21 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
texture.UpdateTexImage();
|
||||
var presentationTime = _clock.GetPresentationTime(texture.Timestamp, Java.Lang.JavaSystem.NanoTime());
|
||||
if (presentationTime is null) return;
|
||||
if (_blackout)
|
||||
{
|
||||
if (!_previewOnly)
|
||||
{
|
||||
GLES20.GlBindFramebuffer(GLES20.GlFramebuffer, 0);
|
||||
GLES20.GlViewport(0, 0, _outputWidth, _outputHeight);
|
||||
GLES20.GlClearColor(0, 0, 0, 1);
|
||||
GLES20.GlClear(GLES20.GlColorBufferBit);
|
||||
CheckGl();
|
||||
if (!EGLExt.EglPresentationTimeANDROID(_display, _window, presentationTime.Value)
|
||||
|| !EGL14.EglSwapBuffers(_display, _window))
|
||||
throw EglError("Envoi d'une image noire a l'encodeur");
|
||||
}
|
||||
return;
|
||||
}
|
||||
texture.GetTransformMatrix(_textureMatrix);
|
||||
var remainingRotation = CameraFrameOrientation.GetRemainingRotation(_rotation, _textureMatrix);
|
||||
if (_remainingRotation != remainingRotation)
|
||||
|
|
@ -204,6 +332,14 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
$"Camera texture transform: [{FirstTextureTransform}], sensor/display rotation={_rotation}, remaining rotation={remainingRotation}");
|
||||
}
|
||||
GLES20.GlUniformMatrix4fv(_matrixLocation, 1, false, _textureMatrix, 0);
|
||||
if (_segmenter is not null) UpdatePersonMask();
|
||||
if (_blurProgram != 0)
|
||||
{
|
||||
GLES20.GlUseProgram(_blurProgram);
|
||||
BindAttribute("position", _vertices!);
|
||||
BindAttribute("textureCoordinate", _coordinates!);
|
||||
GLES20.GlUniformMatrix4fv(_blurMatrixLocation, 1, false, _textureMatrix, 0);
|
||||
}
|
||||
if (!_previewOnly)
|
||||
{
|
||||
SetViewport(_outputWidth, _outputHeight);
|
||||
|
|
@ -215,11 +351,25 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
throw EglError("Envoi de l'image a l'encodeur");
|
||||
}
|
||||
DrawPreview();
|
||||
if (_blurProgram != 0)
|
||||
{
|
||||
GLES20.GlUseProgram(_program);
|
||||
BindAttribute("position", _vertices!);
|
||||
BindAttribute("textureCoordinate", _coordinates!);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_rendering = false;
|
||||
_failed(new InvalidOperationException("Rotation de la capture Android impossible.", ex));
|
||||
if (_blackout)
|
||||
{
|
||||
_rendering = false;
|
||||
global::Android.Util.Log.Error("PostIt.Podcast", $"Arret de la capture impossible : {ex}");
|
||||
}
|
||||
else
|
||||
{
|
||||
_blackout = true;
|
||||
_failed(new InvalidOperationException("Traitement de la capture Android impossible.", ex));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -315,14 +465,14 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
throw new InvalidOperationException($"Compilation du shader camera impossible : {error}");
|
||||
}
|
||||
|
||||
private static int CreateProgram()
|
||||
private static int CreateProgram(string fragmentSource)
|
||||
{
|
||||
var vertex = CreateShader(GLES20.GlVertexShader, VertexShader);
|
||||
var fragment = 0;
|
||||
var program = 0;
|
||||
try
|
||||
{
|
||||
fragment = CreateShader(GLES20.GlFragmentShader, FragmentShader);
|
||||
fragment = CreateShader(GLES20.GlFragmentShader, fragmentSource);
|
||||
program = GLES20.GlCreateProgram();
|
||||
GLES20.GlAttachShader(program, vertex);
|
||||
GLES20.GlAttachShader(program, fragment);
|
||||
|
|
@ -384,6 +534,13 @@ internal sealed class AndroidCameraFrameRotator : IDisposable
|
|||
_texture?.Release();
|
||||
_texture?.Dispose();
|
||||
_listener?.Dispose();
|
||||
_segmenter?.Dispose();
|
||||
_samplingBuffer?.Dispose();
|
||||
_maskBuffer?.Dispose();
|
||||
if (_framebuffer != 0) GLES20.GlDeleteFramebuffers(1, [_framebuffer], 0);
|
||||
if (_samplingTexture != 0) GLES20.GlDeleteTextures(1, [_samplingTexture], 0);
|
||||
if (_maskTexture != 0) GLES20.GlDeleteTextures(1, [_maskTexture], 0);
|
||||
if (_blurProgram != 0) GLES20.GlDeleteProgram(_blurProgram);
|
||||
if (_program != 0) GLES20.GlDeleteProgram(_program);
|
||||
if (_textureId != 0) GLES20.GlDeleteTextures(1, [_textureId], 0);
|
||||
_vertices?.Dispose();
|
||||
|
|
|
|||
|
|
@ -85,9 +85,16 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
|
||||
public static async Task<IPodcastCapture> StartAsync(string camera, string? microphone, string directory, CancellationToken ct)
|
||||
=> await StartCoreAsync(camera, microphone,
|
||||
(stream, token) => new TransportStreamCapture(directory).RunAsync(stream, token), null, null, ct);
|
||||
(stream, token) => new TransportStreamCapture(directory).RunAsync(stream, token), null, null, ct, false);
|
||||
|
||||
public static async Task<IPodcastCapture> StartPreviewAsync(string camera, CancellationToken ct)
|
||||
public static async Task<IPodcastCapture> StartBlurredAsync(string camera, string? microphone, string directory, CancellationToken ct)
|
||||
=> await StartCoreAsync(camera, microphone,
|
||||
(stream, token) => new TransportStreamCapture(directory).RunAsync(stream, token), null, null, ct, true);
|
||||
|
||||
public static Task<IPodcastCapture> StartPreviewAsync(string camera, CancellationToken ct) => StartPreviewAsync(camera, ct, false);
|
||||
public static Task<IPodcastCapture> StartBlurredPreviewAsync(string camera, CancellationToken ct) => StartPreviewAsync(camera, ct, true);
|
||||
|
||||
private static async Task<IPodcastCapture> StartPreviewAsync(string camera, CancellationToken ct, bool blur)
|
||||
{
|
||||
var activity = MainActivity.Current ?? throw new InvalidOperationException("Activite Android indisponible.");
|
||||
await PodcastPermissions.EnsureAsync(activity, false, ct, notifications: false);
|
||||
|
|
@ -99,7 +106,7 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
Platform.PlaybackSuspending += capture.RequestStop;
|
||||
try
|
||||
{
|
||||
await capture.StartNativeAsync(camera, false, null, null, ct);
|
||||
await capture.StartNativeAsync(camera, false, null, null, ct, blur);
|
||||
return capture;
|
||||
}
|
||||
catch
|
||||
|
|
@ -111,11 +118,11 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
|
||||
internal static Task<IPodcastCapture> StartCallAsync(string camera, string microphone,
|
||||
Func<int, CancellationToken, Task> output, Func<Task> stopOutput, CancellationToken ct) =>
|
||||
StartCoreAsync(camera, microphone, null, output, stopOutput, ct);
|
||||
StartCoreAsync(camera, microphone, null, output, stopOutput, ct, false);
|
||||
|
||||
private static async Task<IPodcastCapture> StartCoreAsync(string camera, string? microphone,
|
||||
Func<Stream, CancellationToken, Task>? output,
|
||||
Func<int, CancellationToken, Task>? pipeOutput, Func<Task>? stopOutput, CancellationToken ct)
|
||||
Func<int, CancellationToken, Task>? pipeOutput, Func<Task>? stopOutput, CancellationToken ct, bool blur)
|
||||
{
|
||||
await RequestPermissionsAsync(microphone, ct);
|
||||
var activity = MainActivity.Current ?? throw new InvalidOperationException("Activite Android indisponible.");
|
||||
|
|
@ -133,7 +140,7 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
try
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await capture.StartNativeAsync(camera, microphone is not null, output, pipeOutput, ct);
|
||||
await capture.StartNativeAsync(camera, microphone is not null, output, pipeOutput, ct, blur);
|
||||
return capture;
|
||||
}
|
||||
catch
|
||||
|
|
@ -145,16 +152,16 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
|
||||
private async Task StartNativeAsync(string cameraId, bool microphone,
|
||||
Func<Stream, CancellationToken, Task>? output,
|
||||
Func<int, CancellationToken, Task>? pipeOutput, CancellationToken ct)
|
||||
Func<int, CancellationToken, Task>? pipeOutput, CancellationToken ct, bool blur)
|
||||
{
|
||||
await _stopGate.WaitAsync(ct);
|
||||
try { await ConfigureNativeAsync(cameraId, microphone, output, pipeOutput, ct); }
|
||||
try { await ConfigureNativeAsync(cameraId, microphone, output, pipeOutput, ct, blur); }
|
||||
finally { _stopGate.Release(); }
|
||||
}
|
||||
|
||||
private async Task ConfigureNativeAsync(string cameraId, bool microphone,
|
||||
Func<Stream, CancellationToken, Task>? output,
|
||||
Func<int, CancellationToken, Task>? pipeOutput, CancellationToken ct)
|
||||
Func<int, CancellationToken, Task>? pipeOutput, CancellationToken ct, bool blur)
|
||||
{
|
||||
var manager = GetManager(_context);
|
||||
using var characteristics = manager.GetCameraCharacteristics(cameraId);
|
||||
|
|
@ -205,7 +212,8 @@ internal sealed class AndroidPodcastCapture : IPodcastCapture
|
|||
}
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (_stopRequested) throw new System.OperationCanceledException("Demarrage de la capture annule.");
|
||||
_rotator = await AndroidCameraFrameRotator.CreateAsync(_surface, size.Width, size.Height, rotation, Fail);
|
||||
_rotator = await AndroidCameraFrameRotator.CreateAsync(_surface, size.Width, size.Height, rotation, Fail,
|
||||
blur ? _context : null);
|
||||
_preview = new AndroidCapturePreview(_rotator);
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (_stopRequested) throw new System.OperationCanceledException("Demarrage de la capture annule.");
|
||||
|
|
|
|||
72
src/PostIt/PostIt.Android/Services/PodcastPersonSegmenter.cs
Normal file
72
src/PostIt/PostIt.Android/Services/PodcastPersonSegmenter.cs
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Android.Content;
|
||||
using Microsoft.ML.OnnxRuntime;
|
||||
|
||||
namespace PostIt.Android.Services;
|
||||
|
||||
internal interface IPodcastPersonSegmenter : IDisposable
|
||||
{
|
||||
byte[] Segment(byte[] rgba);
|
||||
}
|
||||
|
||||
internal sealed class PodcastPersonSegmenter : IPodcastPersonSegmenter
|
||||
{
|
||||
internal const int Size = 192;
|
||||
private const int Pixels = Size * Size;
|
||||
private readonly InferenceSession _session;
|
||||
private readonly RunOptions _runOptions = new();
|
||||
private readonly float[] _input = new float[3 * Pixels];
|
||||
private readonly byte[] _mask = new byte[Pixels];
|
||||
private readonly string _inputName;
|
||||
|
||||
internal PodcastPersonSegmenter(Context context)
|
||||
{
|
||||
using var asset = context.Assets?.Open("PodcastBackground/pphumanseg.onnx")
|
||||
?? throw new InvalidOperationException("Modele de segmentation Android introuvable.");
|
||||
using var model = new MemoryStream();
|
||||
asset.CopyTo(model);
|
||||
using var options = new SessionOptions { IntraOpNumThreads = 2, InterOpNumThreads = 1 };
|
||||
_session = new InferenceSession(model.ToArray(), options);
|
||||
_inputName = _session.InputNames.Single();
|
||||
}
|
||||
|
||||
public byte[] Segment(byte[] rgba)
|
||||
{
|
||||
if (rgba.Length != Pixels * 4) throw new ArgumentException("Image de segmentation invalide.", nameof(rgba));
|
||||
for (var y = 0; y < Size; y++)
|
||||
{
|
||||
for (var x = 0; x < Size; x++)
|
||||
{
|
||||
var pixel = y * Size + x;
|
||||
var source = ((Size - 1 - y) * Size + x) * 4;
|
||||
_input[pixel] = rgba[source] / 127.5f - 1;
|
||||
_input[Pixels + pixel] = rgba[source + 1] / 127.5f - 1;
|
||||
_input[2 * Pixels + pixel] = rgba[source + 2] / 127.5f - 1;
|
||||
}
|
||||
}
|
||||
using var input = OrtValue.CreateTensorValueFromMemory(_input, [1, 3, Size, Size]);
|
||||
using var outputs = _session.Run(_runOptions, new Dictionary<string, OrtValue> { [_inputName] = input }, _session.OutputNames);
|
||||
var result = outputs[0].GetTensorDataAsSpan<float>();
|
||||
if (result.Length != 2 * Pixels) throw new InvalidDataException("Sortie du modele de segmentation invalide.");
|
||||
for (var y = 0; y < Size; y++)
|
||||
for (var x = 0; x < Size; x++)
|
||||
{
|
||||
var pixel = y * Size + x;
|
||||
var background = result[pixel];
|
||||
var person = result[Pixels + pixel];
|
||||
if (!float.IsFinite(background) || !float.IsFinite(person))
|
||||
throw new InvalidDataException("Sortie du modele de segmentation non finie.");
|
||||
_mask[(Size - 1 - y) * Size + x] = person > background ? (byte)255 : (byte)0;
|
||||
}
|
||||
return _mask;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_session.Dispose();
|
||||
_runOptions.Dispose();
|
||||
}
|
||||
}
|
||||
171
src/PostIt/PostIt.Tests/AndroidPodcastBlurTestActivity.cs
Normal file
171
src/PostIt/PostIt.Tests/AndroidPodcastBlurTestActivity.cs
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
#if ANDROID
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.Versioning;
|
||||
using System.Threading.Tasks;
|
||||
using Android.App;
|
||||
using Android.Content;
|
||||
using Android.Graphics;
|
||||
using Android.Media;
|
||||
using Android.OS;
|
||||
using PostIt.Android.Services;
|
||||
using Path = System.IO.Path;
|
||||
|
||||
namespace PostIt.Tests;
|
||||
|
||||
[Activity(Name = "fr.pschneider.postit.PodcastBlurTestActivity", Exported = true)]
|
||||
[SupportedOSPlatform("android26.0")]
|
||||
public sealed class AndroidPodcastBlurTestActivity : Activity
|
||||
{
|
||||
protected override async void OnCreate(Bundle? state)
|
||||
{
|
||||
base.OnCreate(state);
|
||||
var result = Path.Combine(FilesDir!.AbsolutePath, "podcast-blur-test-result.txt");
|
||||
File.WriteAllText(result, "RUNNING");
|
||||
try
|
||||
{
|
||||
await VerifyAsync();
|
||||
await VerifyRealModelAsync();
|
||||
await VerifyFailureAsync();
|
||||
File.WriteAllText(result, "PASS\nsegmentation=ONNX\nreal-model-preview=verified\npreview=verified\nencoded=created\nfailure=closed\nsource=synthetic");
|
||||
}
|
||||
catch (Exception ex) { File.WriteAllText(result, $"FAIL\n{ex}"); }
|
||||
finally { Finish(); }
|
||||
}
|
||||
|
||||
private async Task VerifyAsync()
|
||||
{
|
||||
const int width = 640, height = 480;
|
||||
using (var segmenter = new PodcastPersonSegmenter(this))
|
||||
{
|
||||
var mask = segmenter.Segment(new byte[PodcastPersonSegmenter.Size * PodcastPersonSegmenter.Size * 4]);
|
||||
if (mask.Length != PodcastPersonSegmenter.Size * PodcastPersonSegmenter.Size)
|
||||
throw new InvalidOperationException("Masque de segmentation invalide.");
|
||||
}
|
||||
var path = Path.Combine(FilesDir!.AbsolutePath, "podcast-blur-test.ts");
|
||||
#pragma warning disable CS0618
|
||||
using var recorder = OperatingSystem.IsAndroidVersionAtLeast(31) ? new MediaRecorder(this) : new MediaRecorder();
|
||||
#pragma warning restore CS0618
|
||||
recorder.SetVideoSource(VideoSource.Surface);
|
||||
recorder.SetOutputFormat(OutputFormat.Mpeg2Ts);
|
||||
recorder.SetVideoSize(width, height);
|
||||
recorder.SetVideoFrameRate(30);
|
||||
recorder.SetVideoEncodingBitRate(1200000);
|
||||
recorder.SetVideoEncoder(VideoEncoder.H264);
|
||||
recorder.SetOutputFile(path);
|
||||
recorder.Prepare();
|
||||
using var surface = recorder.Surface ?? throw new InvalidOperationException("Encodeur indisponible.");
|
||||
using var preview = ImageReader.NewInstance(width, height, (ImageFormatType)Format.Rgba8888, 3)
|
||||
?? throw new InvalidOperationException("Apercu indisponible.");
|
||||
Exception? failure = null;
|
||||
using var rotator = await AndroidCameraFrameRotator.CreateAsync(surface, width, height, 0,
|
||||
error => failure = error, this, _ => new SplitMask());
|
||||
await rotator.SetPreviewAsync(preview.Surface, width, height, error => failure = error);
|
||||
recorder.Start();
|
||||
await rotator.StartAsync();
|
||||
using var paint = new Paint();
|
||||
var frames = 0;
|
||||
Task? stop = null;
|
||||
for (var frame = 0; frame < 60; frame++)
|
||||
{
|
||||
if (failure is not null) throw failure;
|
||||
if (frame == 40) stop = Task.Run(recorder.Stop);
|
||||
if (stop?.IsCompleted == true) break;
|
||||
using var canvas = rotator.CameraSurface.LockCanvas(null)
|
||||
?? throw new InvalidOperationException("Camera synthetique indisponible.");
|
||||
try
|
||||
{
|
||||
for (var x = 0; x < width; x += 8)
|
||||
{
|
||||
paint.Color = x / 8 % 2 == 0 ? Color.Black : Color.White;
|
||||
canvas.DrawRect(x, 0, x + 8, height, paint);
|
||||
}
|
||||
}
|
||||
finally { rotator.CameraSurface.UnlockCanvasAndPost(canvas); }
|
||||
await Task.Delay(70);
|
||||
using var image = preview.AcquireLatestImage();
|
||||
if (image is not null)
|
||||
{
|
||||
var plane = image.GetPlanes()![0];
|
||||
var buffer = plane.Buffer!;
|
||||
int Red(int x, int y) => buffer.Get(y * plane.RowStride + x * plane.PixelStride) & 255;
|
||||
var background = Math.Abs(Red(100, 240) - Red(108, 240));
|
||||
var foreground = Math.Abs(Red(500, 240) - Red(508, 240));
|
||||
if (frame > 5 && (background > 120 || foreground < 150))
|
||||
throw new InvalidOperationException($"Apercu incorrect : decor={background}, personne={foreground}.");
|
||||
frames++;
|
||||
image.Close();
|
||||
}
|
||||
}
|
||||
if (failure is not null) throw failure;
|
||||
if (frames < 5) throw new InvalidOperationException($"Apercu insuffisant : {frames} images.");
|
||||
await (stop ?? throw new InvalidOperationException("Arret de l'encodeur non demande."))
|
||||
.WaitAsync(TimeSpan.FromSeconds(15));
|
||||
if (new FileInfo(path).Length == 0) throw new InvalidOperationException("Video encodee vide.");
|
||||
}
|
||||
|
||||
private sealed class SplitMask : IPodcastPersonSegmenter
|
||||
{
|
||||
private readonly byte[] _mask = new byte[PodcastPersonSegmenter.Size * PodcastPersonSegmenter.Size];
|
||||
|
||||
public SplitMask()
|
||||
{
|
||||
for (var y = 0; y < PodcastPersonSegmenter.Size; y++)
|
||||
for (var x = PodcastPersonSegmenter.Size / 2; x < PodcastPersonSegmenter.Size; x++)
|
||||
_mask[y * PodcastPersonSegmenter.Size + x] = 255;
|
||||
}
|
||||
public byte[] Segment(byte[] rgba) => _mask;
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private async Task VerifyFailureAsync()
|
||||
{
|
||||
using var preview = ImageReader.NewInstance(640, 480, (ImageFormatType)Format.Rgba8888, 3)
|
||||
?? throw new InvalidOperationException("Apercu indisponible.");
|
||||
var failed = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using var rotator = await AndroidCameraFrameRotator.CreateAsync(null, 640, 480, 0,
|
||||
error => failed.TrySetResult(error), this, _ => new FailingMask());
|
||||
await rotator.SetPreviewAsync(preview.Surface, preview.Width, preview.Height,
|
||||
error => failed.TrySetResult(error));
|
||||
await rotator.StartAsync();
|
||||
using var canvas = rotator.CameraSurface.LockCanvas(null)
|
||||
?? throw new InvalidOperationException("Camera synthetique indisponible.");
|
||||
try { canvas.DrawColor(Color.Red); }
|
||||
finally { rotator.CameraSurface.UnlockCanvasAndPost(canvas); }
|
||||
await failed.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
using var image = preview.AcquireLatestImage();
|
||||
if (image is not null) throw new InvalidOperationException("Une image non filtree a ete affichee apres l'echec.");
|
||||
}
|
||||
|
||||
private async Task VerifyRealModelAsync()
|
||||
{
|
||||
using var preview = ImageReader.NewInstance(640, 480, (ImageFormatType)Format.Rgba8888, 3)
|
||||
?? throw new InvalidOperationException("Apercu indisponible.");
|
||||
Exception? failure = null;
|
||||
using var rotator = await AndroidCameraFrameRotator.CreateAsync(null, 640, 480, 0,
|
||||
error => failure = error, this);
|
||||
await rotator.SetPreviewAsync(preview.Surface, preview.Width, preview.Height, error => failure = error);
|
||||
await rotator.StartAsync();
|
||||
var frames = 0;
|
||||
for (var i = 0; i < 15; i++)
|
||||
{
|
||||
if (failure is not null) throw failure;
|
||||
using var canvas = rotator.CameraSurface.LockCanvas(null)
|
||||
?? throw new InvalidOperationException("Camera synthetique indisponible.");
|
||||
try { canvas.DrawColor(Color.Gray); }
|
||||
finally { rotator.CameraSurface.UnlockCanvasAndPost(canvas); }
|
||||
await Task.Delay(100);
|
||||
using var image = preview.AcquireLatestImage();
|
||||
if (image is not null) { frames++; image.Close(); }
|
||||
}
|
||||
if (failure is not null) throw failure;
|
||||
if (frames < 2) throw new InvalidOperationException($"Apercu avec modele reel insuffisant : {frames} images.");
|
||||
}
|
||||
|
||||
private sealed class FailingMask : IPodcastPersonSegmenter
|
||||
{
|
||||
public byte[] Segment(byte[] rgba) => throw new InvalidOperationException("Echec de segmentation simule.");
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
|
@ -29,6 +29,8 @@ public sealed class PodcastPreferencesTests
|
|||
vm.Title = "Mon emission habituelle";
|
||||
Assert.False(vm.KeepScreenOnDuringCapture);
|
||||
vm.KeepScreenOnDuringCapture = true;
|
||||
Assert.False(vm.BlurPodcastBackground);
|
||||
vm.BlurPodcastBackground = true;
|
||||
vm.DetectDevicesCommand.Execute(null);
|
||||
Assert.Equal("rear", vm.SelectedCamera?.Value);
|
||||
Assert.Equal("", vm.SelectedMicrophone?.Value);
|
||||
|
|
@ -41,6 +43,7 @@ public sealed class PodcastPreferencesTests
|
|||
Assert.Equal("rear", restored.PodcastCameraId);
|
||||
Assert.Equal("", restored.PodcastMicrophoneId);
|
||||
Assert.True(restored.PodcastKeepScreenOn);
|
||||
Assert.True(restored.PodcastBlurBackground);
|
||||
Platform.GetPodcastCaptureDevices = () => throw new InvalidOperationException("Must use saved devices.");
|
||||
await using var otherApi = new PreferencesApi(restored, new TokenStore(path + ".tokens"));
|
||||
var other = new PodcastsPageViewModel(new PodcastApiClient(otherApi, () => restored.ApiUrl), otherApi, session);
|
||||
|
|
@ -50,13 +53,16 @@ public sealed class PodcastPreferencesTests
|
|||
Assert.Equal("", other.SelectedMicrophone?.Value);
|
||||
Assert.Equal(vm.Title, other.Title);
|
||||
Assert.True(other.KeepScreenOnDuringCapture);
|
||||
Assert.True(other.BlurPodcastBackground);
|
||||
restored.PodcastCameraId = "front";
|
||||
restored.PodcastMicrophoneId = "usb";
|
||||
restored.PodcastKeepScreenOn = false;
|
||||
restored.PodcastBlurBackground = false;
|
||||
await other.ActivateAsync();
|
||||
Assert.Equal("front", other.SelectedCamera?.Value);
|
||||
Assert.Equal("usb", other.SelectedMicrophone?.Value);
|
||||
Assert.False(other.KeepScreenOnDuringCapture);
|
||||
Assert.False(other.BlurPodcastBackground);
|
||||
|
||||
Platform.GetPodcastCaptureDevices = () => new([], []);
|
||||
other.DetectDevicesCommand.Execute(null);
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ public static class Platform
|
|||
public static Func<string?, CancellationToken, Task>? RequestPodcastPermissionsAsync { get; set; }
|
||||
public static Func<string, string?, string, CancellationToken, Task<IPodcastCapture>>? CreatePodcastCaptureAsync { get; set; }
|
||||
public static Func<string, CancellationToken, Task<IPodcastCapture>>? CreatePodcastPreviewAsync { get; set; }
|
||||
public static Func<string, string?, string, CancellationToken, Task<IPodcastCapture>>? CreateBlurredPodcastCaptureAsync { get; set; }
|
||||
public static Func<string, CancellationToken, Task<IPodcastCapture>>? CreateBlurredPodcastPreviewAsync { get; set; }
|
||||
|
||||
/// <summary>Platform-specific video surface inside the shared video page.</summary>
|
||||
public static Func<MediaPlayer, bool, Control>? CreateVideoSurface { get; set; }
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
private Control? _capturePreview;
|
||||
private bool _restoringPreferences;
|
||||
private bool _keepScreenOnDuringCapture;
|
||||
private bool _blurPodcastBackground;
|
||||
private Guid? _obsPodcastId;
|
||||
private int _obsVersion;
|
||||
[ObservableProperty] public partial string ObsServerUrl { get; set; } = "";
|
||||
|
|
@ -42,6 +43,22 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
[ObservableProperty] public partial string ObsStatus { get; set; } = "";
|
||||
private IDisposable? _screenAwake;
|
||||
public bool SupportsCaptureScreenAwake => OperatingSystem.IsAndroid();
|
||||
public bool SupportsPodcastBackgroundBlur => Platform.CreateBlurredPodcastCaptureAsync is not null
|
||||
&& Platform.CreateBlurredPodcastPreviewAsync is not null;
|
||||
public bool CanChangePodcastBackgroundBlur => !IsCapturing && !IsBusy;
|
||||
public bool BlurPodcastBackground
|
||||
{
|
||||
get => _blurPodcastBackground;
|
||||
set
|
||||
{
|
||||
if (!SetProperty(ref _blurPodcastBackground, value)) return;
|
||||
if (!_restoringPreferences)
|
||||
{
|
||||
_api.Settings.PodcastBlurBackground = value;
|
||||
SavePreferences();
|
||||
}
|
||||
}
|
||||
}
|
||||
public bool KeepScreenOnDuringCapture
|
||||
{
|
||||
get => _keepScreenOnDuringCapture;
|
||||
|
|
@ -123,6 +140,7 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
{
|
||||
if (!SetProperty(ref _isBroadcasting, value)) return;
|
||||
OnPropertyChanged(nameof(IsCapturing));
|
||||
OnPropertyChanged(nameof(CanChangePodcastBackgroundBlur));
|
||||
UpdateScreenAwake();
|
||||
UpdateCommands();
|
||||
}
|
||||
|
|
@ -131,7 +149,12 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
public bool IsBusy
|
||||
{
|
||||
get => _busy;
|
||||
private set { if (SetProperty(ref _busy, value)) UpdateCommands(); }
|
||||
private set
|
||||
{
|
||||
if (!SetProperty(ref _busy, value)) return;
|
||||
OnPropertyChanged(nameof(CanChangePodcastBackgroundBlur));
|
||||
UpdateCommands();
|
||||
}
|
||||
}
|
||||
|
||||
public CaptureDevice? SelectedCamera
|
||||
|
|
@ -245,6 +268,7 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
&& p.OwnerId == _ownerId && !IsBroadcasting && !IsBusy);
|
||||
_title = _api.Settings.PodcastDefaultTitle;
|
||||
_keepScreenOnDuringCapture = _api.Settings.PodcastKeepScreenOn;
|
||||
_blurPodcastBackground = _api.Settings.PodcastBlurBackground;
|
||||
_restoringPreferences = true;
|
||||
try
|
||||
{
|
||||
|
|
@ -352,6 +376,7 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
try
|
||||
{
|
||||
KeepScreenOnDuringCapture = _api.Settings.PodcastKeepScreenOn;
|
||||
BlurPodcastBackground = _api.Settings.PodcastBlurBackground;
|
||||
Cameras.Clear();
|
||||
Microphones.Clear();
|
||||
foreach (var device in _api.Settings.PodcastCameras) Cameras.Add(device);
|
||||
|
|
@ -477,7 +502,11 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
ct.ThrowIfCancellationRequested();
|
||||
if (_previewCapture is not null) return;
|
||||
var camera = SelectedCamera?.Value ?? throw new InvalidOperationException("Choisissez une camera.");
|
||||
capture = Platform.CreatePodcastPreviewAsync is { } createPreview
|
||||
var previewFactory = BlurPodcastBackground && OperatingSystem.IsAndroid()
|
||||
? Platform.CreateBlurredPodcastPreviewAsync
|
||||
?? throw new InvalidOperationException("Le flou d'arriere-plan n'est pas disponible sur cet appareil.")
|
||||
: Platform.CreatePodcastPreviewAsync;
|
||||
capture = previewFactory is { } createPreview
|
||||
? await createPreview(camera, ct)
|
||||
: new LibVlcPodcastCapture(
|
||||
vlc => new CameraStreamer(vlc).CreateCaptureMedia(camera, null), null, preview: true);
|
||||
|
|
@ -506,6 +535,7 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
{
|
||||
OnPropertyChanged(nameof(IsPreviewing));
|
||||
OnPropertyChanged(nameof(IsCapturing));
|
||||
OnPropertyChanged(nameof(CanChangePodcastBackgroundBlur));
|
||||
UpdateScreenAwake();
|
||||
UpdateCommands();
|
||||
}
|
||||
|
|
@ -553,7 +583,11 @@ public partial class PodcastsPageViewModel : ViewModelBase
|
|||
var microphone = string.IsNullOrWhiteSpace(SelectedMicrophone?.Value) ? null : SelectedMicrophone.Value;
|
||||
if (Platform.RequestPodcastPermissionsAsync is { } requestPermissions)
|
||||
await requestPermissions(microphone, _pageCancellation.Token);
|
||||
_broadcaster = Platform.CreatePodcastCaptureAsync is { } createCapture
|
||||
var captureFactory = BlurPodcastBackground && OperatingSystem.IsAndroid()
|
||||
? Platform.CreateBlurredPodcastCaptureAsync
|
||||
?? throw new InvalidOperationException("Le flou d'arriere-plan n'est pas disponible sur cet appareil.")
|
||||
: Platform.CreatePodcastCaptureAsync;
|
||||
_broadcaster = captureFactory is { } createCapture
|
||||
? await PodcastBroadcaster.StartAsync(_client, Title,
|
||||
(directory, ct) => createCapture(camera, microphone, directory, ct), _pageCancellation.Token, SelectedPodcast?.Id)
|
||||
: await PodcastBroadcaster.StartAsync(_client, Title, camera, microphone, _pageCancellation.Token, SelectedPodcast?.Id);
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ public partial class Settings : ViewModelBase
|
|||
[ObservableProperty]
|
||||
public partial bool PodcastKeepScreenOn { get; set; }
|
||||
[ObservableProperty]
|
||||
public partial bool PodcastBlurBackground { get; set; }
|
||||
[ObservableProperty]
|
||||
public partial string? LanCallAddress { get; set; }
|
||||
[ObservableProperty]
|
||||
public partial bool AndroidKeepChatConnected { get; set; }
|
||||
|
|
@ -169,6 +171,7 @@ public partial class Settings : ViewModelBase
|
|||
partial void OnPodcastCameraIdChanged(string? value) => MarkDirty();
|
||||
partial void OnPodcastMicrophoneIdChanged(string? value) => MarkDirty();
|
||||
partial void OnPodcastKeepScreenOnChanged(bool value) => MarkDirty();
|
||||
partial void OnPodcastBlurBackgroundChanged(bool value) => MarkDirty();
|
||||
partial void OnLanCallAddressChanged(string? value) => MarkDirty();
|
||||
partial void OnAndroidKeepChatConnectedChanged(bool value) => MarkDirty();
|
||||
partial void OnChatRoomsByAccountChanged(Dictionary<string, string[]> value) => MarkDirty();
|
||||
|
|
@ -435,6 +438,7 @@ public partial class Settings : ViewModelBase
|
|||
PodcastCameraId = settings.PodcastCameraId;
|
||||
PodcastMicrophoneId = settings.PodcastMicrophoneId;
|
||||
PodcastKeepScreenOn = settings.PodcastKeepScreenOn;
|
||||
PodcastBlurBackground = settings.PodcastBlurBackground;
|
||||
LanCallAddress = settings.LanCallAddress;
|
||||
AndroidKeepChatConnected = settings.AndroidKeepChatConnected;
|
||||
ChatRoomsByAccount = settings.ChatRoomsByAccount ?? new();
|
||||
|
|
|
|||
|
|
@ -90,6 +90,12 @@
|
|||
<DataTemplate x:DataType="services:CaptureDevice"><TextBlock Text="{Binding Name}" /></DataTemplate>
|
||||
</ComboBox.ItemTemplate>
|
||||
</ComboBox>
|
||||
<CheckBox Content="Flouter le decor de la camera (Android)"
|
||||
IsVisible="{Binding SupportsPodcastBackgroundBlur}"
|
||||
IsEnabled="{Binding CanChangePodcastBackgroundBlur}"
|
||||
IsChecked="{Binding BlurPodcastBackground, Mode=TwoWay}" />
|
||||
<TextBlock Text="Le masque automatique peut laisser apparaitre des details du decor. Verifiez l'apercu avant de diffuser."
|
||||
IsVisible="{Binding SupportsPodcastBackgroundBlur}" TextWrapping="Wrap" />
|
||||
<WrapPanel>
|
||||
<Button Content="Previsualiser la camera" Command="{Binding PreviewCommand}" Margin="0,0,8,8" />
|
||||
<Button Content="Arreter la previsualisation" Command="{Binding StopPreviewCommand}" Margin="0,0,8,8" />
|
||||
|
|
|
|||
Loading…
Reference in a new issue