diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 3edb7b053..bc2c36299 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1868,9 +1868,8 @@ def unified_timestamp(date_str, day_first=True, with_milliseconds=False):
 
     for expression in date_formats(day_first):
         with contextlib.suppress(ValueError):
-            dt = datetime.datetime.strptime(date_str, expression) + datetime.timedelta(hours=pm_delta)
-            dt_tz_aware = dt.replace(tzinfo=datetime.timezone(timezone))
-            return dt_tz_aware.timestamp() if with_milliseconds else int(dt_tz_aware.timestamp())
+            dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
+            return calendar.timegm(dt.timetuple()) + (dt.microsecond / 1e6 if with_milliseconds else 0)
 
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple: