copilot/OpenSeeFace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

RealSense Support

Browse source
This commit is contained in:
CSGNS 2025-08-29 12:31:20 +09:00
parent dee5bb7606
commit 5ce9fd3684
7 changed files with 800 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

52
.github/actions/upload-dated-artifact/action.yaml vendored
View file

@ -1,26 +1,26 @@
name: Upload dated artifact
description: Uploads an artifact with a datetime
inputs:
name:
description: The name of the artifact
required: true
path:
description: The path to the file/folder to upload
required: true
runs:
using: composite
steps:
# https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows
- name: Get current datetime
id: date
shell: bash
run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')"
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.name }}_${{ steps.date.outputs.date }}
path: ${{ inputs.path }}
if-no-files-found: error
name: Upload dated artifact
description: Uploads an artifact with a datetime
inputs:
name:
description: The name of the artifact
required: true
path:
description: The path to the file/folder to upload
required: true
runs:
using: composite
steps:
# https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows
- name: Get current datetime
id: date
shell: bash
run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')"
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.name }}_${{ steps.date.outputs.date }}
path: ${{ inputs.path }}
if-no-files-found: error

174
.github/workflows/create-binaries.yaml vendored
View file

@ -1,87 +1,87 @@
name: Package using PyInstaller
on:
push:
tags:
- v**
workflow_dispatch:
jobs:
build-windows:
env:
dist-path: dist/OpenSeeFace
runs-on: windows-latest
steps:
- name: Pull OpenSeeFace
uses: actions/checkout@v3
- name: Create binary
shell: cmd
run: |
choco install vcredist140
xcopy c:\Windows\System32\msvcp140.dll .
xcopy c:\Windows\System32\vcomp140.dll .
xcopy c:\Windows\System32\concrt140.dll .
xcopy c:\Windows\System32\vccorlib140.dll .
make_exe.bat
- name: Move files for uploading
shell: bash
run: |
# Prevents the parent dist/ directory from being included
mv dist/facetracker ${{ env.dist-path }}
cp -r models ${{ env.dist-path }}
- name: Upload artifact
uses: ./.github/actions/upload-dated-artifact
with:
name: windows
path: dist # Add the parent directory to prevent a tarbomb
# Files must be packed into a tar in order for file permissions to be preserved
build-unix:
env:
artifact-path: OpenSeeFace
tar-path: OpenSeeFace.tar
strategy:
fail-fast: false
matrix:
include:
- name: linux
os: ubuntu-latest
- name: mac
os: macos-latest
runs-on: ${{ matrix.os }}
steps:
- name: Pull OpenSeeFace
uses: actions/checkout@v3
- name: Create binary
shell: bash
run: |
chmod +x make_exe.sh
./make_exe.sh
- name: Move files for uploading
shell: bash
run: |
# Prevents the parent dist/ directory from being included
mv dist/facetracker ${{ env.artifact-path }}
cp -r models ${{ env.artifact-path }}
- name: Fix permissions from PyInstaller
shell: bash
run: |
find ${{ env.artifact-path }} -type f -exec chmod -x {} +
chmod +x ${{ env.artifact-path }}/facetracker
- name: Pack files into tar
shell: bash
run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }}
- name: Upload artifact
uses: ./.github/actions/upload-dated-artifact
with:
name: ${{ matrix.name }}
path: ${{ env.tar-path }}
name: Package using PyInstaller
on:
push:
tags:
- v**
workflow_dispatch:
jobs:
build-windows:
env:
dist-path: dist/OpenSeeFace
runs-on: windows-latest
steps:
- name: Pull OpenSeeFace
uses: actions/checkout@v3
- name: Create binary
shell: cmd
run: |
choco install vcredist140
xcopy c:\Windows\System32\msvcp140.dll .
xcopy c:\Windows\System32\vcomp140.dll .
xcopy c:\Windows\System32\concrt140.dll .
xcopy c:\Windows\System32\vccorlib140.dll .
make_exe.bat
- name: Move files for uploading
shell: bash
run: |
# Prevents the parent dist/ directory from being included
mv dist/facetracker ${{ env.dist-path }}
cp -r models ${{ env.dist-path }}
- name: Upload artifact
uses: ./.github/actions/upload-dated-artifact
with:
name: windows
path: dist # Add the parent directory to prevent a tarbomb
# Files must be packed into a tar in order for file permissions to be preserved
build-unix:
env:
artifact-path: OpenSeeFace
tar-path: OpenSeeFace.tar
strategy:
fail-fast: false
matrix:
include:
- name: linux
os: ubuntu-latest
- name: mac
os: macos-latest
runs-on: ${{ matrix.os }}
steps:
- name: Pull OpenSeeFace
uses: actions/checkout@v3
- name: Create binary
shell: bash
run: |
chmod +x make_exe.sh
./make_exe.sh
- name: Move files for uploading
shell: bash
run: |
# Prevents the parent dist/ directory from being included
mv dist/facetracker ${{ env.artifact-path }}
cp -r models ${{ env.artifact-path }}
- name: Fix permissions from PyInstaller
shell: bash
run: |
find ${{ env.artifact-path }} -type f -exec chmod -x {} +
chmod +x ${{ env.artifact-path }}/facetracker
- name: Pack files into tar
shell: bash
run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }}
- name: Upload artifact
uses: ./.github/actions/upload-dated-artifact
with:
name: ${{ matrix.name }}
path: ${{ env.tar-path }}

202
Licenses/pyrealsense2.txt Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

177
facetracker.py
View file

@ -47,6 +47,8 @@ parser.add_argument("--face-id-offset", type=int, help="When set, this offset is
parser.add_argument("--repeat-video", type=int, help="When set to 1 and a video file was specified with -c, the tracker will loop the video until interrupted", default=0)
parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
parser.add_argument("--benchmark", type=int, help="When set to 1, the different tracking models are benchmarked, starting with the best and ending with the fastest and with gaze tracking disabled for models with negative IDs", default=0)
parser.add_argument("--realsense", type=int, help="When set to 1, uses pyrealsense2 for camera input", default=0)
if os.name == 'nt':
parser.add_argument("--use-dshowcapture", type=int, help="When set to 1, libdshowcapture will be used for video input instead of OpenCV", default=1)
parser.add_argument("--blackmagic-options", type=str, help="When set, this additional option string is passed to the blackmagic capture library", default=None)
@ -159,11 +161,31 @@ use_dshowcapture_flag = False
if os.name == 'nt':
dcap = args.dcap
use_dshowcapture_flag = True if args.use_dshowcapture == 1 else False
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
input_reader = InputReader(
args.capture,
args.raw_rgb,
args.width,
args.height,
fps,
use_dshowcapture=(args.use_dshowcapture == 1) if os.name == 'nt' else False,
dcap=args.dcap if os.name == 'nt' else None,
use_realsense=(args.realsense == 1)
)
if args.dcap == -1 and type(input_reader) == DShowCaptureReader:
fps = min(fps, input_reader.device.get_fps())
else:
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps)
input_reader = InputReader(
args.capture,
args.raw_rgb,
args.width,
args.height,
fps,
use_dshowcapture=(args.use_dshowcapture == 1) if os.name == 'nt' else False,
dcap=args.dcap if os.name == 'nt' else None,
use_realsense=(args.realsense == 1)
)
if type(input_reader.reader) == VideoReader:
fps = 0
@ -205,9 +227,29 @@ try:
need_reinit = 0
failures = 0
source_name = input_reader.name
# 디버그 정보를 저장할 변수들
debug_info = {
'inference_time': 0,
'faces_count': 0,
'frame_info': '',
'tracking_status': 'No faces detected',
'data_sent': False
}
while repeat or input_reader.is_open():
if not input_reader.is_open() or need_reinit == 1:
input_reader = InputReader(args.capture, args.raw_rgb, args.width, args.height, fps, use_dshowcapture=use_dshowcapture_flag, dcap=dcap)
input_reader = InputReader(
args.capture,
args.raw_rgb,
args.width,
args.height,
fps,
use_dshowcapture=(args.use_dshowcapture == 1) if os.name == 'nt' else False,
dcap=args.dcap if os.name == 'nt' else None,
use_realsense=(args.realsense == 1)
)
if input_reader.name != source_name:
print(f"Failed to reinitialize camera and got {input_reader.name} instead of {source_name}.")
sys.exit(1)
@ -250,17 +292,32 @@ try:
tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 and args.model != -1 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False, try_hard=args.try_hard == 1)
if args.video_out is not None:
out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), args.video_fps, (width * args.video_scale, height * args.video_scale))
try:
inference_start = time.perf_counter()
faces = tracker.predict(frame)
inference_time = (time.perf_counter() - inference_start)
# 디버그 정보 업데이트
debug_info['inference_time'] = inference_time * 1000 # ms로 변환
debug_info['faces_count'] = len(faces)
if len(faces) > 0:
inference_time = (time.perf_counter() - inference_start)
total_tracking_time += inference_time
tracking_time += inference_time / len(faces)
tracking_frames += 1
debug_info['tracking_status'] = f"Tracking {len(faces)} face(s)"
else:
debug_info['tracking_status'] = "No faces detected"
# 프레임 정보 업데이트 (30프레임마다)
if frame_count % 30 == 0:
debug_info['frame_info'] = f"Frame {frame_count}: {frame.shape} {frame.dtype} Min:{frame.min()} Max:{frame.max()}"
packet = bytearray()
detected = False
face_details = [] # 각 얼굴의 세부 정보를 저장
for face_num, f in enumerate(faces):
f = copy.copy(f)
f.id += args.face_id_offset
@ -268,9 +325,18 @@ try:
f.eye_blink = [1, 1]
right_state = "O" if f.eye_blink[0] > 0.30 else "-"
left_state = "O" if f.eye_blink[1] > 0.30 else "-"
if args.silent == 0:
print(f"Confidence[{f.id}]: {f.conf:.4f} / 3D fitting error: {f.pnp_error:.4f} / Eyes: {left_state}, {right_state}")
# 얼굴 정보를 리스트에 저장 (화면에 표시용)
face_details.append({
'id': f.id,
'conf': f.conf,
'pnp_error': f.pnp_error,
'left_eye': left_state,
'right_eye': right_state
})
detected = True
if not f.success:
pts_3d = np.zeros((70, 3), np.float32)
packet.extend(bytearray(struct.pack("d", now)))
@ -292,14 +358,17 @@ try:
packet.extend(bytearray(struct.pack("f", f.translation[1])))
packet.extend(bytearray(struct.pack("f", f.translation[2])))
if log is not None:
log.write(f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
for (x,y,c) in f.lms:
log.write(
f"{frame_count},{now},{width},{height},{fps},{face_num},{f.id},{f.eye_blink[0]},{f.eye_blink[1]},{f.conf},{f.success},{f.pnp_error},{f.quaternion[0]},{f.quaternion[1]},{f.quaternion[2]},{f.quaternion[3]},{f.euler[0]},{f.euler[1]},{f.euler[2]},{f.rotation[0]},{f.rotation[1]},{f.rotation[2]},{f.translation[0]},{f.translation[1]},{f.translation[2]}")
for (x, y, c) in f.lms:
packet.extend(bytearray(struct.pack("f", c)))
if args.visualize > 1:
frame = cv2.putText(frame, str(f.id), (int(f.bbox[0]), int(f.bbox[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,255))
frame = cv2.putText(frame, str(f.id), (int(f.bbox[0]), int(f.bbox[1])), cv2.FONT_HERSHEY_SIMPLEX,
0.75, (255, 0, 255))
if args.visualize > 2:
frame = cv2.putText(frame, f"{f.conf:.4f}", (int(f.bbox[0] + 18), int(f.bbox[1] - 6)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255))
for pt_num, (x,y,c) in enumerate(f.lms):
frame = cv2.putText(frame, f"{f.conf:.4f}", (int(f.bbox[0] + 18), int(f.bbox[1] - 6)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))
for pt_num, (x, y, c) in enumerate(f.lms):
packet.extend(bytearray(struct.pack("f", y)))
packet.extend(bytearray(struct.pack("f", x)))
if log is not None:
@ -312,7 +381,8 @@ try:
y = int(y + 0.5)
if args.visualize != 0 or out is not None:
if args.visualize > 3:
frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (255,255,0))
frame = cv2.putText(frame, str(pt_num), (int(y), int(x)), cv2.FONT_HERSHEY_SIMPLEX, 0.25,
(255, 255, 0))
color = (0, 255, 0)
if pt_num >= 66:
color = (255, 255, 0)
@ -320,10 +390,12 @@ try:
cv2.circle(frame, (y, x), 1, color, -1)
if args.pnp_points != 0 and (args.visualize != 0 or out is not None) and f.rotation is not None:
if args.pnp_points > 1:
projected = cv2.projectPoints(f.face_3d[0:66], f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)
projected = cv2.projectPoints(f.face_3d[0:66], f.rotation, f.translation, tracker.camera,
tracker.dist_coeffs)
else:
projected = cv2.projectPoints(f.contour, f.rotation, f.translation, tracker.camera, tracker.dist_coeffs)
for [(x,y)] in projected[0]:
projected = cv2.projectPoints(f.contour, f.rotation, f.translation, tracker.camera,
tracker.dist_coeffs)
for [(x, y)] in projected[0]:
x = int(x + 0.5)
y = int(y + 0.5)
if not (x < 0 or y < 0 or x >= height or y >= width):
@ -337,7 +409,7 @@ try:
x -= 1
if not (x < 0 or y < 0 or x >= height or y >= width):
frame[int(x), int(y)] = (0, 255, 255)
for (x,y,z) in f.pts_3d:
for (x, y, z) in f.pts_3d:
packet.extend(bytearray(struct.pack("f", x)))
packet.extend(bytearray(struct.pack("f", -y)))
packet.extend(bytearray(struct.pack("f", -z)))
@ -355,13 +427,33 @@ try:
log.write("\r\n")
log.flush()
# 데이터 전송 상태 업데이트
if detected and len(faces) < 40:
sock.sendto(packet, (target_ip, target_port))
debug_info['data_sent'] = True
else:
debug_info['data_sent'] = False
# OpenCV 창에 간단한 디버그 정보 표시 (args.visualize != 0일 때만)
if args.visualize != 0:
y_pos = 20
line_height = 18
# 기본 정보를 한 줄로 압축
main_info = f"Inference:{debug_info['inference_time']:.1f}ms | {debug_info['tracking_status']} | Data:{'OK' if debug_info['data_sent'] else 'NO'} | Frame:{frame_count}"
cv2.putText(frame, main_info, (10, y_pos), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 0), 1)
# 얼굴별 정보를 간단하게 표시
for i, face_info in enumerate(face_details):
face_text = f"F{face_info['id']}:C{face_info['conf']:.2f} E{face_info['pnp_error']:.2f} [{face_info['left_eye']}{face_info['right_eye']}]"
cv2.putText(frame, face_text, (10, y_pos + (i + 1) * line_height),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (255, 255, 0), 1)
if out is not None:
video_frame = frame
if args.video_scale != 1:
video_frame = cv2.resize(frame, (width * args.video_scale, height * args.video_scale), interpolation=cv2.INTER_NEAREST)
video_frame = cv2.resize(frame, (width * args.video_scale, height * args.video_scale),
interpolation=cv2.INTER_NEAREST)
out.write(video_frame)
if args.video_scale != 1:
del video_frame
@ -369,51 +461,6 @@ try:
if args.visualize != 0:
cv2.imshow('OpenSeeFace Visualization', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
if args.dump_points != "" and faces is not None and len(faces) > 0:
np.set_printoptions(threshold=sys.maxsize, precision=15)
pairs = [
(0, 16),
(1, 15),
(2, 14),
(3, 13),
(4, 12),
(5, 11),
(6, 10),
(7, 9),
(17, 26),
(18, 25),
(19, 24),
(20, 23),
(21, 22),
(31, 35),
(32, 34),
(36, 45),
(37, 44),
(38, 43),
(39, 42),
(40, 47),
(41, 46),
(48, 52),
(49, 51),
(56, 54),
(57, 53),
(58, 62),
(59, 61),
(65, 63)
]
points = copy.copy(faces[0].face_3d)
for a, b in pairs:
x = (points[a, 0] - points[b, 0]) / 2.0
y = (points[a, 1] + points[b, 1]) / 2.0
z = (points[a, 2] + points[b, 2]) / 2.0
points[a, 0] = x
points[b, 0] = -x
points[[a, b], 1] = y
points[[a, b], 2] = z
points[[8, 27, 28, 29, 33, 50, 55, 60, 64], 0] = 0.0
points[30, :] = 0.0
with open(args.dump_points, "w") as fh:
fh.write(repr(points))
break
failures = 0
except Exception as e:
@ -421,7 +468,9 @@ try:
if args.silent == 0:
print("Quitting")
break
traceback.print_exc()
# 예외 발생 시에도 화면에 표시
if args.visualize != 0:
cv2.putText(frame, f"Error: {str(e)[:50]}...", (15, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)
failures += 1
if failures > 30:
break

252
input_reader.py
View file

@ -3,12 +3,25 @@ import sys
import os
import cv2
import numpy as np
import escapi
import dshowcapture
import time
import traceback
import gc
# Libraries exclusive to the Windows environment may cause errors when imported, so they are wrapped in try-except.
try:
import escapi
except ImportError:
escapi = None
try:
import dshowcapture
except ImportError:
dshowcapture = None
try:
import pyrealsense2 as rs
except ImportError:
rs = None
class VideoReader():
def __init__(self, capture, camera=False):
if os.name == 'nt' and camera:
@ -19,17 +32,24 @@ class VideoReader():
print("The video source cannot be opened")
sys.exit(0)
self.name = str(capture)
def is_open(self):
return self.cap.isOpened()
def is_ready(self):
return True
def read(self):
return self.cap.read()
def close(self):
self.cap.release()
class EscapiReader(VideoReader):
def __init__(self, capture, width, height, fps):
if escapi is None:
raise ImportError("escapi module is not available.")
self.device = None
self.width = width
self.height = height
@ -39,10 +59,13 @@ class EscapiReader(VideoReader):
self.name = str(escapi.device_name(self.device).decode('utf8', 'surrogateescape'))
self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
escapi.do_capture(self.device)
def is_open(self):
return True
def is_ready(self):
return escapi.is_capture_done(self.device)
def read(self):
if escapi.is_capture_done(self.device):
image = escapi.read(self.device, self.width, self.height, self.buffer)
@ -50,11 +73,16 @@ class EscapiReader(VideoReader):
return True, image
else:
return False, None
def close(self):
escapi.deinit_camera(self.device)
if escapi:
escapi.deinit_camera(self.device)
class DShowCaptureReader(VideoReader):
def __init__(self, capture, width, height, fps, use_dshowcapture=True, dcap=None):
if dshowcapture is None:
raise ImportError("dshowcapture module is not available.")
self.device = None
self.width = width
self.height = height
@ -81,12 +109,16 @@ class DShowCaptureReader(VideoReader):
self.width = self.device.width
self.height = self.device.height
self.fps = self.device.fps
print(f"Camera: \"{self.name}\" Capability ID: {dcap} Resolution: {self.device.width}x{self.device.height} Frame rate: {self.device.fps} Colorspace: {self.device.colorspace} Internal: {self.device.colorspace_internal} Flipped: {self.device.flipped}")
print(
f"Camera: \"{self.name}\" Capability ID: {dcap} Resolution: {self.device.width}x{self.device.height} Frame rate: {self.device.fps} Colorspace: {self.device.colorspace} Internal: {self.device.colorspace_internal} Flipped: {self.device.flipped}")
self.timeout = 1000
def is_open(self):
return self.device.capturing()
def is_ready(self):
return self.device.capturing()
def read(self):
img = None
try:
@ -98,8 +130,11 @@ class DShowCaptureReader(VideoReader):
return False, None
else:
return True, img
def close(self):
self.device.destroy_capture()
if self.device:
self.device.destroy_capture()
class OpenCVReader(VideoReader):
def __init__(self, capture, width, height, fps):
@ -112,30 +147,38 @@ class OpenCVReader(VideoReader):
self.cap.set(3, width)
self.cap.set(4, height)
self.cap.set(38, 1)
def is_open(self):
return super(OpenCVReader, self).is_open()
def is_ready(self):
return super(OpenCVReader, self).is_ready()
def read(self):
return super(OpenCVReader, self).read()
def close(self):
super(OpenCVReader, self).close()
class RawReader:
def __init__(self, width, height):
self.width = int(width)
self.height = int(height)
if self.width < 1 or self.height < 1:
print("No acceptable size was given for reading raw RGB frames.")
sys.exit(0)
self.len = self.width * self.height * 3
self.open = True
def is_open(self):
return self.open
def is_ready(self):
return True
def read(self):
frame = bytearray()
read_bytes = 0
@ -144,55 +187,185 @@ class RawReader:
read_bytes += len(bytes)
frame.extend(bytes)
return True, np.frombuffer(frame, dtype=np.uint8).reshape((self.height, self.width, 3))
def close(self):
self.open = False
class RealSenseReader:
def __init__(self, width, height, fps):
if rs is None:
raise ImportError(
"pyrealsense2 module is not available. Please install it with 'pip install pyrealsense2'.")
self.pipeline = rs.pipeline()
self.config = rs.config()
self.name = "RealSense Camera"
self.width = width
self.height = height
self.fps = fps
self.is_pipeline_started = False
# Set Color stream
self.config.enable_stream(rs.stream.color, width, height, rs.format.bgr8, fps)
# Set Depth stream
self.config.enable_stream(rs.stream.depth, width, height, rs.format.z16, fps)
try:
# realsense pipeline start
self.profile = self.pipeline.start(self.config)
self.is_pipeline_started = True
# Wait a moment for the camera to stabilize
time.sleep(2)
# Discard a few frames to warm up the camera
for _ in range(10):
try:
frames = self.pipeline.wait_for_frames(timeout_ms=1000)
if frames:
break
except:
continue
print(f"RealSense camera started successfully. Resolution: {width}x{height}, FPS: {fps}")
except Exception as e:
print(f"Error starting RealSense pipeline: {e}")
self.is_pipeline_started = False
raise RuntimeError("RealSense camera failed to start.")
def is_open(self):
return self.is_pipeline_started and self.pipeline is not None
def is_ready(self):
return self.is_pipeline_started
def read(self):
if not self.is_pipeline_started:
return False, None
try:
# Set a timeout to wait for frames
frames = self.pipeline.wait_for_frames(timeout_ms=5000) # 5초 타임아웃
if not frames:
print("No frames received from RealSense camera")
return False, None
color_frame = frames.get_color_frame()
depth_frame = frames.get_depth_frame()
if not color_frame:
print("No color frame available")
return False, None
# Convert frame data to numpy array
frame = np.asanyarray(color_frame.get_data())
# Check Frame Size
if frame is None or frame.size == 0:
print("Empty frame received")
return False, None
# Check Frame Type
if len(frame.shape) != 3 or frame.shape[2] != 3:
print(f"Unexpected frame shape: {frame.shape}")
return False, None
# Check BGR
if frame.dtype != np.uint8:
frame = frame.astype(np.uint8)
return True, frame
except rs.error as e:
print(f"RealSense error: {e}")
return False, None
except Exception as e:
print(f"General error reading RealSense frame: {e}")
traceback.print_exc()
return False, None
def close(self):
if self.is_pipeline_started and self.pipeline:
try:
self.pipeline.stop()
self.is_pipeline_started = False
print("RealSense camera stopped.")
except Exception as e:
print(f"Error stopping RealSense pipeline: {e}")
def try_int(s):
try:
return int(s)
except:
return None
def test_reader(reader):
got_any = 0
try:
for i in range(30):
if not reader.is_ready():
time.sleep(0.02)
time.sleep(0.1) # add waiting time for realsense
continue
ret, frame = reader.read()
if not ret:
time.sleep(0.02)
print("No frame")
time.sleep(0.1)
print(f"No frame at attempt {i}")
else:
print("Got frame")
print(f"Got frame {i + 1}, shape: {frame.shape if frame is not None else 'None'}")
got_any += 1
if got_any > 10:
if got_any > 5: # Fewer frames are enough for testing
break
if reader.is_open():
return got_any > 0
print("Fail")
success = got_any > 0
print(f"Test reader result: {success}, got {got_any} frames")
return success
print("Reader not open - Fail")
return False
except:
except Exception as e:
print(f"Test reader exception: {e}")
traceback.print_exc()
print("Except")
return False
class InputReader():
def __init__(self, capture, raw_rgb, width, height, fps, use_dshowcapture=False, dcap=None):
def __init__(self, capture, raw_rgb, width, height, fps, use_dshowcapture=False, dcap=None, use_realsense=False):
self.reader = None
self.name = str(capture)
try:
if raw_rgb > 0:
print("Using raw RGB input...")
self.reader = RawReader(width, height)
elif use_realsense:
print("Attempting to use RealSense camera...")
self.reader = RealSenseReader(width, height, fps)
self.name = self.reader.name
# RealSense reader test
print("Testing RealSense reader...")
if test_reader(self.reader):
print("RealSense reader test passed")
return
else:
print("RealSense reader test failed")
raise RuntimeError("RealSense camera test failed")
elif os.path.exists(capture):
print(f"Using video file: {capture}")
self.reader = VideoReader(capture)
elif capture == str(try_int(capture)):
if os.name == 'nt':
# Try with DShowCapture
# try DShowCapture
good = True
name = ""
try:
if use_dshowcapture:
if use_dshowcapture and dshowcapture:
self.reader = DShowCaptureReader(int(capture), width, height, fps, dcap=dcap)
name = self.reader.name
good = test_reader(self.reader)
@ -205,46 +378,55 @@ class InputReader():
good = False
if good:
return
# Try with Escapi
# try Escapi
good = True
try:
print(f"DShowCapture failed. Falling back to escapi for device {name}.", file=sys.stderr)
escapi.init()
devices = escapi.count_capture_devices()
found = None
for i in range(devices):
escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape'))
if name == escapi_name:
found = i
if found is None:
good = False
if escapi:
print(f"DShowCapture failed. Falling back to escapi for device {name}.", file=sys.stderr)
escapi.init()
devices = escapi.count_capture_devices()
found = None
for i in range(devices):
escapi_name = str(escapi.device_name(i).decode('utf8', 'surrogateescape'))
if name == escapi_name:
found = i
if found is None:
good = False
else:
print(f"Found device {name} as {i}.", file=sys.stderr)
self.reader = EscapiReader(found, width, height, fps)
good = test_reader(self.reader)
else:
print(f"Found device {name} as {i}.", file=sys.stderr)
self.reader = EscapiReader(found, width, height, fps)
good = test_reader(self.reader)
good = False
except:
print("Escapi exception: ")
traceback.print_exc()
good = False
if good:
return
# Try with OpenCV
print(f"Escapi failed. Falling back to OpenCV. If this fails, please change your camera settings.", file=sys.stderr)
# try opencv
print(f"Escapi failed. Falling back to OpenCV. If this fails, please change your camera settings.",
file=sys.stderr)
self.reader = OpenCVReader(int(capture), width, height, fps)
self.name = self.reader.name
else:
self.reader = OpenCVReader(int(capture), width, height, fps)
except Exception as e:
print("Error: " + str(e))
traceback.print_exc()
if self.reader is None or not self.reader.is_open():
print("There was no valid input.")
sys.exit(0)
def is_open(self):
return self.reader.is_open()
def is_ready(self):
return self.reader.is_ready()
def read(self):
return self.reader.read()
def close(self):
self.reader.close()
self.reader.close()

1
pyproject.toml
View file

@ -11,6 +11,7 @@ numpy = "^1.21.3"
opencv-python = "^4.5.4"
Pillow = "^8.4.0"
onnxruntime = "^1.9.0"
pyrealsense2 = "^2.56.5"
[tool.poetry.dev-dependencies]

188
tracker.py
View file

@ -397,7 +397,8 @@ class FaceInfo():
return pts_3d
def adjust_3d(self):
def adjust_3d(self, depth_frame=None):
if self.conf < 0.4 or self.pnp_error > 300:
return
@ -408,8 +409,27 @@ class FaceInfo():
update_type = -1
d_o = np.ones((66,))
d_c = np.ones((66,))
depth_weights = np.ones((66,)) # RealSense 깊이 가중치
# RealSense 깊이 데이터가 있을 때 가중치 계산
if depth_frame is not None:
try:
for i in range(66):
x, y = int(self.lms[i][0]), int(self.lms[i][1])
if 0 <= x < depth_frame.get_width() and 0 <= y < depth_frame.get_height():
actual_depth = depth_frame.get_distance(x, y)
if actual_depth > 0.1 and actual_depth < 10.0:
estimated_depth = abs(self.pts_3d[i][2])
if estimated_depth > 0:
# 깊이 오차가 클수록 가중치 증가 (더 많은 조정 필요)
depth_error = abs(actual_depth - estimated_depth)
depth_weights[i] = min(2.0, 1.0 + depth_error / estimated_depth)
except Exception as e:
if not self.tracker.silent:
print(f"Depth weight calculation failed: {e}")
for runs in range(max_runs):
r = 1.0 + np.random.random_sample((66,3)) * 0.02 - 0.01
r = 1.0 + np.random.random_sample((66, 3)) * 0.02 - 0.01
r[30, :] = 1.0
if self.euler[0] > -165 and self.euler[0] < 145:
continue
@ -423,30 +443,74 @@ class FaceInfo():
# Enable only one side of the points, depending on direction
elif self.euler[1] < -10:
update_type = 1
r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58, 59, 65], 2] = 1.0
eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 42, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64]
r[[0, 1, 2, 3, 4, 5, 6, 7, 17, 18, 19, 20, 21, 31, 32, 36, 37, 38, 39, 40, 41, 48, 49, 56, 57, 58,
59, 65], 2] = 1.0
eligible = [8, 9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 42, 43,
44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 60, 61, 62, 63, 64]
else:
update_type = 1
r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 61, 62, 63], 2] = 1.0
eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 31, 32, 33, 36, 37, 38, 39, 40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65]
r[[9, 10, 11, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 34, 35, 42, 43, 44, 45, 46, 47, 51, 52, 53,
54, 61, 62, 63], 2] = 1.0
eligible = [0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 21, 27, 28, 29, 31, 32, 33, 36, 37, 38, 39,
40, 41, 48, 49, 50, 55, 56, 57, 58, 59, 60, 64, 65]
if self.limit_3d_adjustment:
eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:, abs(update_type - 1)] + self.update_count_delta)[0]
eligible = np.nonzero(self.update_counts[:, update_type] < self.update_counts[:,
abs(update_type - 1)] + self.update_count_delta)[
0]
if eligible.shape[0] <= 0:
break
if runs == 0:
updated = copy.copy(self.face_3d[0:66])
o_projected = np.ones((66,2))
o_projected[eligible] = np.squeeze(np.array(cv2.projectPoints(self.face_3d[eligible], self.rotation, self.translation, self.tracker.camera, self.tracker.dist_coeffs)[0]), 1)
o_projected = np.ones((66, 2))
o_projected[eligible] = np.squeeze(np.array(
cv2.projectPoints(self.face_3d[eligible], self.rotation, self.translation, self.tracker.camera,
self.tracker.dist_coeffs)[0]), 1)
c = updated * r
c_projected = np.zeros((66,2))
c_projected[eligible] = np.squeeze(np.array(cv2.projectPoints(c[eligible], self.rotation, self.translation, self.tracker.camera, self.tracker.dist_coeffs)[0]), 1)
c_projected = np.zeros((66, 2))
c_projected[eligible] = np.squeeze(np.array(
cv2.projectPoints(c[eligible], self.rotation, self.translation, self.tracker.camera,
self.tracker.dist_coeffs)[0]), 1)
changed = False
d_o[eligible] = np.linalg.norm(o_projected[eligible] - self.lms[eligible, 0:2], axis=1)
d_c[eligible] = np.linalg.norm(c_projected[eligible] - self.lms[eligible, 0:2], axis=1)
indices = np.nonzero(d_c < d_o)[0]
# RealSense 깊이 데이터가 있을 때 깊이 오차도 고려
if depth_frame is not None:
try:
depth_error_o = np.zeros(66)
depth_error_c = np.zeros(66)
for i in eligible:
x, y = int(self.lms[i][0]), int(self.lms[i][1])
if 0 <= x < depth_frame.get_width() and 0 <= y < depth_frame.get_height():
actual_depth = depth_frame.get_distance(x, y)
if actual_depth > 0.1 and actual_depth < 10.0:
# 원본과 후보 모델의 깊이 오차 계산
orig_depth = abs(self.face_3d[i][2])
cand_depth = abs(c[i][2])
depth_error_o[i] = abs(actual_depth - orig_depth) * depth_weights[i]
depth_error_c[i] = abs(actual_depth - cand_depth) * depth_weights[i]
# 2D 투영 오차와 깊이 오차를 결합 (깊이 오차에 가중치 적용)
depth_weight_factor = 0.3 # 깊이 오차의 영향도 조절
combined_error_o = d_o + depth_error_o * depth_weight_factor
combined_error_c = d_c + depth_error_c * depth_weight_factor
indices = np.nonzero(combined_error_c < combined_error_o)[0]
except Exception as e:
# 깊이 데이터 처리 중 오류 발생 시 기존 방식 사용
indices = np.nonzero(d_c < d_o)[0]
if not self.tracker.silent:
print(f"Depth-enhanced 3D adjustment failed, using 2D only: {e}")
else:
# RealSense가 없으면 기존 방식 사용
indices = np.nonzero(d_c < d_o)[0]
if indices.shape[0] > 0:
if self.limit_3d_adjustment:
indices = np.intersect1d(indices, eligible)
@ -461,15 +525,30 @@ class FaceInfo():
break
if changed_any:
# Update weighted by point confidence
weights = np.zeros((66,3))
# Update weighted by point confidence and depth reliability
weights = np.zeros((66, 3))
weights[:, :] = self.lms[0:66, 2:3]
weights[weights > 0.7] = 1.0
weights = 1.0 - weights
# RealSense가 있을 때 깊이 신뢰도도 가중치에 반영
if depth_frame is not None:
try:
for i in range(66):
x, y = int(self.lms[i][0]), int(self.lms[i][1])
if 0 <= x < depth_frame.get_width() and 0 <= y < depth_frame.get_height():
actual_depth = depth_frame.get_distance(x, y)
if actual_depth > 0.1 and actual_depth < 10.0:
# 깊이 데이터가 신뢰할 만하면 가중치 조정
weights[i] *= 0.8 # 더 많은 업데이트 허용
except Exception:
pass
update_indices = np.arange(0, 66)
if self.limit_3d_adjustment:
update_indices = np.nonzero(self.update_counts[:, update_type] <= self.update_count_max)[0]
self.face_3d[update_indices] = self.face_3d[update_indices] * weights[update_indices] + updated[update_indices] * (1. - weights[update_indices])
self.face_3d[update_indices] = self.face_3d[update_indices] * weights[update_indices] + updated[
update_indices] * (1. - weights[update_indices])
self.update_contour()
self.pts_3d = self.normalize_pts3d(self.pts_3d)
@ -667,7 +746,7 @@ class Tracker():
self.wait_count = 0
self.scan_every = scan_every
self.bbox_growth = bbox_growth
self.silent = silent
self.silent = False
self.try_hard = try_hard
self.res = 224.
@ -761,23 +840,62 @@ class Tracker():
avg_conf = part_avg
return (avg_conf, np.array(lms))
def estimate_depth(self, face_info):
lms = np.concatenate((face_info.lms, np.array([[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]], [face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float32)), 0)
def estimate_depth(self, face_info, depth_frame=None):
lms = np.concatenate((face_info.lms, np.array(
[[face_info.eye_state[0][1], face_info.eye_state[0][2], face_info.eye_state[0][3]],
[face_info.eye_state[1][1], face_info.eye_state[1][2], face_info.eye_state[1][3]]], np.float32)), 0)
image_pts = np.array(lms)[face_info.contour_pts, 0:2]
success = False
if face_info.rotation is not None:
success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=np.transpose(face_info.rotation), tvec=np.transpose(face_info.translation), flags=cv2.SOLVEPNP_ITERATIVE)
success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera,
self.dist_coeffs, useExtrinsicGuess=True,
rvec=np.transpose(face_info.rotation),
tvec=np.transpose(face_info.translation),
flags=cv2.SOLVEPNP_ITERATIVE)
else:
rvec = np.array([0, 0, 0], np.float32)
tvec = np.array([0, 0, 0], np.float32)
success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera, self.dist_coeffs, useExtrinsicGuess=True, rvec=rvec, tvec=tvec, flags=cv2.SOLVEPNP_ITERATIVE)
success, face_info.rotation, face_info.translation = cv2.solvePnP(face_info.contour, image_pts, self.camera,
self.dist_coeffs, useExtrinsicGuess=True,
rvec=rvec, tvec=tvec,
flags=cv2.SOLVEPNP_ITERATIVE)
rotation = face_info.rotation
translation = face_info.translation
pts_3d = np.zeros((70,3), np.float32)
# RealSense 깊이 데이터가 있을 때 거리 보정 적용
if depth_frame is not None and success:
try:
# 코 끝점 (인덱스 30)의 실제 깊이 측정
nose_x, nose_y = int(lms[30][0]), int(lms[30][1])
# 이미지 경계 확인
if 0 <= nose_x < depth_frame.get_width() and 0 <= nose_y < depth_frame.get_height():
actual_depth = depth_frame.get_distance(nose_x, nose_y)
if actual_depth > 0.1 and actual_depth < 10.0: # 유효한 깊이 범위 (0.1m ~ 10m)
# Translation vector의 Z값을 실제 깊이로 보정
depth_scale = actual_depth / (translation[2][0] if translation[2][0] != 0 else 1.0)
# 급격한 변화를 방지하기 위한 스무딩
if hasattr(face_info, 'prev_depth_scale') and face_info.prev_depth_scale is not None:
alpha = 0.7 # 스무딩 팩터
depth_scale = alpha * face_info.prev_depth_scale + (1 - alpha) * depth_scale
face_info.prev_depth_scale = depth_scale
# Translation vector 조정
translation = translation * depth_scale
face_info.translation = translation
except Exception as e:
# 깊이 데이터 처리 중 오류 발생 시 기존 방식 유지
if not self.silent:
print(f"RealSense depth correction failed: {e}")
pts_3d = np.zeros((70, 3), np.float32)
if not success:
face_info.rotation = np.array([0.0, 0.0, 0.0], np.float32)
face_info.translation = np.array([0.0, 0.0, 0.0], np.float32)
@ -793,12 +911,13 @@ class Tracker():
t_reference = t_reference.dot(self.camera.transpose())
t_depth = t_reference[:, 2]
t_depth[t_depth == 0] = 0.000001
t_depth_e = np.expand_dims(t_depth[:],1)
t_depth_e = np.expand_dims(t_depth[:], 1)
t_reference = t_reference[:] / t_depth_e
pts_3d[0:66] = np.stack([lms[0:66,0], lms[0:66,1], np.ones((66,))], 1) * t_depth_e[0:66]
pts_3d[0:66] = (pts_3d[0:66].dot(self.inverse_camera.transpose()) - face_info.translation).dot(inverse_rotation.transpose())
pnp_error = np.power(lms[0:17,0:2] - t_reference[0:17,0:2], 2).sum()
pnp_error += np.power(lms[30,0:2] - t_reference[30,0:2], 2).sum()
pts_3d[0:66] = np.stack([lms[0:66, 0], lms[0:66, 1], np.ones((66,))], 1) * t_depth_e[0:66]
pts_3d[0:66] = (pts_3d[0:66].dot(self.inverse_camera.transpose()) - face_info.translation).dot(
inverse_rotation.transpose())
pnp_error = np.power(lms[0:17, 0:2] - t_reference[0:17, 0:2], 2).sum()
pnp_error += np.power(lms[30, 0:2] - t_reference[30, 0:2], 2).sum()
if np.isnan(pnp_error):
pnp_error = 9999999.
for i, pt in enumerate(face_info.face_3d[66:70]):
@ -820,13 +939,13 @@ class Tracker():
pts_3d[69] = pt_3d
continue
if i == 0:
d1 = np.linalg.norm(lms[66,0:2] - lms[36,0:2])
d2 = np.linalg.norm(lms[66,0:2] - lms[39,0:2])
d1 = np.linalg.norm(lms[66, 0:2] - lms[36, 0:2])
d2 = np.linalg.norm(lms[67, 0:2] - lms[39, 0:2])
d = d1 + d2
pt = (pts_3d[36] * d1 + pts_3d[39] * d2) / d
if i == 1:
d1 = np.linalg.norm(lms[67,0:2] - lms[42,0:2])
d2 = np.linalg.norm(lms[67,0:2] - lms[45,0:2])
d1 = np.linalg.norm(lms[67, 0:2] - lms[42, 0:2])
d2 = np.linalg.norm(lms[67, 0:2] - lms[45, 0:2])
d = d1 + d2
pt = (pts_3d[42] * d1 + pts_3d[45] * d2) / d
if i < 2:
@ -834,12 +953,12 @@ class Tracker():
reference = reference + face_info.translation
reference = self.camera.dot(reference)
depth = reference[2]
pt_3d = np.array([lms[66+i][0] * depth, lms[66+i][1] * depth, depth], np.float32)
pt_3d = np.array([lms[66 + i][0] * depth, lms[66 + i][1] * depth, depth], np.float32)
pt_3d = self.inverse_camera.dot(pt_3d)
pt_3d = pt_3d - face_info.translation
pt_3d = inverse_rotation.dot(pt_3d)
pts_3d[66+i,:] = pt_3d[:]
pts_3d[np.isnan(pts_3d).any(axis=1)] = np.array([0.,0.,0.], dtype=np.float32)
pts_3d[66 + i, :] = pt_3d[:]
pts_3d[np.isnan(pts_3d).any(axis=1)] = np.array([0., 0., 0.], dtype=np.float32)
pnp_error = np.sqrt(pnp_error / (2.0 * image_pts.shape[0]))
if pnp_error > 300:
@ -851,13 +970,14 @@ class Tracker():
face_info.face_3d = copy.copy(self.face_3d)
face_info.rotation = None
face_info.translation = np.array([0.0, 0.0, 0.0], np.float32)
face_info.update_counts = np.zeros((66,2))
face_info.update_counts = np.zeros((66, 2))
face_info.update_contour()
else:
face_info.fail_count = 0
euler = cv2.RQDecomp3x3(rmat)[0]
return True, matrix_to_quaternion(rmat), euler, pnp_error, pts_3d, lms
def preprocess(self, im, crop):
x1, y1, x2, y2 = crop