diff --git a/OpsLogging.cs b/OpsLogging.cs
new file mode 100644
index 0000000..eae3bf0
--- /dev/null
+++ b/OpsLogging.cs
@@ -0,0 +1,338 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Text;
+
+namespace HikvisionAttendanceService;
+
+/// Canonical markers for the operational timeline.
+internal static class OpsMarkers
+{
+ public const string Service = "SERVICE";
+ public const string Scope = "SCOPE";
+ public const string Connectivity = "CONNECTIVITY";
+ public const string Attendance = "ATTENDANCE";
+ public const string UserDelete = "USER_DELETE";
+ public const string TemplateDeviceToDb = "TEMPLATE_DEVICE_TO_DB";
+ public const string TemplateDbToDevice = "TEMPLATE_DB_TO_DEVICE";
+}
+
+internal enum ConnectivityFailureStage
+{
+ None,
+ InvalidIp,
+ PingFailed,
+ TcpPortClosed,
+ SdkLoginTimeout,
+ InvalidCredentials,
+ SdkRejected,
+ Unknown
+}
+
+internal sealed class ConnectivityProbeResult
+{
+ public bool Reachable { get; set; }
+ public bool PingChecked { get; set; }
+ public bool PingOk { get; set; }
+ public bool TcpChecked { get; set; }
+ public bool TcpOk { get; set; }
+ public ConnectivityFailureStage FailureStage { get; set; } = ConnectivityFailureStage.None;
+ public string FriendlyReason { get; set; } = "";
+ public uint SdkErrorCode { get; set; }
+}
+
+/// Tracks unreachable devices and writes a human-readable unreachable file (no SDK spam).
+internal sealed class UnreachableDeviceTracker
+{
+ private readonly ConcurrentDictionary _states =
+ new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase);
+ private readonly HikvisionAttendanceWindowsService.FileLogger _logger;
+ private int _headerWritten;
+
+ public UnreachableDeviceTracker(HikvisionAttendanceWindowsService.FileLogger logger)
+ {
+ _logger = logger;
+ }
+
+ public void ReportFailure(
+ string machineId,
+ string machineName,
+ string ip,
+ int port,
+ ConnectivityFailureStage stage,
+ string? friendlyReason)
+ {
+ var key = BuildKey(machineId, ip);
+ var reason = BizFriendlyReasons.FromStage(stage, friendlyReason);
+ var isNew = false;
+ var state = _states.AddOrUpdate(
+ key,
+ _ =>
+ {
+ isNew = true;
+ return new UnreachableState
+ {
+ MachineId = machineId ?? "",
+ MachineName = machineName ?? "",
+ Ip = ip ?? "",
+ Port = port,
+ Stage = stage,
+ Reason = reason
+ };
+ },
+ (_, existing) =>
+ {
+ existing.Stage = stage;
+ existing.Reason = reason;
+ if (!string.IsNullOrWhiteSpace(machineName))
+ existing.MachineName = machineName;
+ return existing;
+ });
+
+ if (!isNew)
+ return; // avoid duplicate spam for the same machine in one run
+
+ if (System.Threading.Interlocked.Exchange(ref _headerWritten, 1) == 0)
+ {
+ _logger.Biz(BizChannel.Unreachable,
+ "UNREACHABLE MACHINES",
+ "");
+ _logger.BizSeparator(BizChannel.Unreachable);
+ }
+
+ var siteLabel = "SITE " + (string.IsNullOrWhiteSpace(state.MachineId) ? "?" : state.MachineId);
+ var name = string.IsNullOrWhiteSpace(state.MachineName) ? "(unnamed terminal)" : state.MachineName;
+ _logger.Biz(BizChannel.Unreachable,
+ siteLabel,
+ "",
+ name,
+ "",
+ state.Ip,
+ "",
+ "Reason :",
+ "",
+ reason,
+ "");
+ _logger.BizSeparator(BizChannel.Unreachable);
+
+ _logger.OpsWarn(OpsMarkers.Connectivity,
+ "UNREACHABLE machine_id=" + state.MachineId + " ip=" + state.Ip + " port=" + port +
+ " stage=" + stage + " reason=\"" + reason + "\"");
+ }
+
+ public void ReportRecovered(string machineId, string machineName, string ip, int port)
+ {
+ var key = BuildKey(machineId, ip);
+ if (!_states.TryRemove(key, out var state))
+ return;
+
+ _logger.Biz(BizChannel.Unreachable,
+ "RECOVERED",
+ "",
+ "SITE " + (string.IsNullOrWhiteSpace(machineId) ? state.MachineId : machineId),
+ "",
+ string.IsNullOrWhiteSpace(machineName) ? state.MachineName : machineName,
+ "",
+ ip ?? state.Ip,
+ "",
+ "Reason :",
+ "",
+ "Device connectivity restored.",
+ "");
+ _logger.BizSeparator(BizChannel.Unreachable);
+ _logger.Ops(OpsMarkers.Connectivity,
+ "RECOVERED machine_id=" + (machineId ?? state.MachineId) + " ip=" + (ip ?? state.Ip) + " port=" + port);
+ }
+
+ private static string BuildKey(string? machineId, string? ip) =>
+ ((machineId ?? "").Trim() + "|" + (ip ?? "").Trim()).ToLowerInvariant();
+
+ private sealed class UnreachableState
+ {
+ public string MachineId = "";
+ public string MachineName = "";
+ public string Ip = "";
+ public int Port;
+ public ConnectivityFailureStage Stage;
+ public string Reason = "";
+ }
+}
+
+/// Pre-login IP / optional ping / TCP diagnostics (does not claim ping unless checked).
+internal static class ConnectivityDiagnostics
+{
+ public static ConnectivityProbeResult ProbeBeforeLogin(string? ip, int port, int tcpTimeoutMs = 3000, bool tryPing = true)
+ {
+ var result = new ConnectivityProbeResult();
+ var rawIp = (ip ?? "").Trim();
+ if (string.IsNullOrWhiteSpace(rawIp) ||
+ !IPAddress.TryParse(rawIp, out var address) ||
+ address.Equals(IPAddress.None) ||
+ address.Equals(IPAddress.Any))
+ {
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.InvalidIp;
+ result.FriendlyReason = "Invalid IP address \"" + rawIp + "\".";
+ return result;
+ }
+
+ if (port <= 0 || port > 65535)
+ {
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.TcpPortClosed;
+ result.FriendlyReason = "Invalid TCP port " + port + ".";
+ return result;
+ }
+
+ if (tryPing)
+ {
+ result.PingChecked = true;
+ try
+ {
+ using var ping = new Ping();
+ var reply = ping.Send(address, 1500);
+ result.PingOk = reply != null && reply.Status == IPStatus.Success;
+ if (!result.PingOk)
+ {
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.PingFailed;
+ result.FriendlyReason = "Ping failed (" + (reply?.Status.ToString() ?? "no reply") +
+ "). Host did not respond to ICMP.";
+ // Still attempt TCP — some networks block ICMP but allow SDK port.
+ }
+ }
+ catch
+ {
+ // Ping not available / blocked by OS policy — do not report ping failure.
+ result.PingChecked = false;
+ result.PingOk = false;
+ }
+ }
+
+ result.TcpChecked = true;
+ try
+ {
+ using var client = new TcpClient();
+ var ar = client.BeginConnect(address, port, null, null);
+ bool ok = ar.AsyncWaitHandle.WaitOne(TimeSpan.FromMilliseconds(Math.Max(500, tcpTimeoutMs)));
+ if (!ok)
+ {
+ try { client.Close(); } catch { /* ignore */ }
+ result.TcpOk = false;
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.TcpPortClosed;
+ result.FriendlyReason = "TCP port " + port + " is closed or blocked (connect timed out).";
+ return result;
+ }
+
+ client.EndConnect(ar);
+ result.TcpOk = client.Connected;
+ }
+ catch (Exception ex)
+ {
+ result.TcpOk = false;
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.TcpPortClosed;
+ result.FriendlyReason = "TCP port " + port + " is closed or blocked (" + ex.GetType().Name + ").";
+ return result;
+ }
+
+ if (!result.TcpOk)
+ {
+ result.Reachable = false;
+ result.FailureStage = ConnectivityFailureStage.TcpPortClosed;
+ result.FriendlyReason = "TCP port " + port + " is closed or blocked.";
+ return result;
+ }
+
+ // TCP open — login may still fail (credentials / SDK). Mark provisional reachable until login.
+ result.Reachable = true;
+ result.FailureStage = ConnectivityFailureStage.None;
+ result.FriendlyReason = "TCP port " + port + " is open.";
+ return result;
+ }
+
+ public static ConnectivityProbeResult ClassifySdkLoginFailure(uint sdkError, ConnectivityProbeResult? prior)
+ {
+ var result = prior ?? new ConnectivityProbeResult();
+ result.Reachable = false;
+ result.SdkErrorCode = sdkError;
+ switch (sdkError)
+ {
+ case 1:
+ case 5:
+ result.FailureStage = ConnectivityFailureStage.InvalidCredentials;
+ result.FriendlyReason = "Invalid credentials (SDK error " + sdkError + ").";
+ break;
+ case 41:
+ case 109:
+ result.FailureStage = ConnectivityFailureStage.SdkLoginTimeout;
+ result.FriendlyReason = "SDK login timed out (SDK error " + sdkError + ").";
+ break;
+ case 7:
+ result.FailureStage = ConnectivityFailureStage.TcpPortClosed;
+ result.FriendlyReason = "SDK connection failed — device unreachable on the SDK port (SDK error 7).";
+ break;
+ default:
+ result.FailureStage = ConnectivityFailureStage.SdkRejected;
+ result.FriendlyReason = "ISAPI/SDK rejected login (SDK error " + sdkError + ").";
+ break;
+ }
+ return result;
+ }
+
+ public static string FormatReachableLine(
+ string machineId,
+ string machineName,
+ string ip,
+ int port,
+ ConnectivityProbeResult probe,
+ bool loginOk)
+ {
+ var reachable = loginOk ? "TRUE" : "FALSE";
+ var sb = new StringBuilder();
+ sb.Append("machine_id=").Append(machineId ?? "");
+ sb.Append(" name=\"").Append(machineName ?? "").Append("\"");
+ sb.Append(" ip=").Append(ip ?? "");
+ sb.Append(" port=").Append(port);
+ sb.Append(" Reachable=").Append(reachable);
+ if (probe.PingChecked)
+ sb.Append(" ping=").Append(probe.PingOk ? "OK" : "FAIL");
+ if (probe.TcpChecked)
+ sb.Append(" tcp=").Append(probe.TcpOk ? "OPEN" : "CLOSED");
+ if (!loginOk)
+ sb.Append(" stage=").Append(probe.FailureStage).Append(" reason=\"").Append(probe.FriendlyReason).Append("\"");
+ else
+ sb.Append(" reason=\"SDK login succeeded\"");
+ return sb.ToString();
+ }
+}
+
+internal enum AttendancePersistOutcome
+{
+ Duplicate,
+ SkippedNoEmployee,
+ DbInserted,
+ DbFailed,
+ PersistedWithoutDb
+}
+
+internal sealed class AttendanceDeviceCycleStats
+{
+ public string DeviceId = "";
+ public string DeviceIp = "";
+ public int EventsReceived;
+ public int ValidPunches;
+ public int SystemEventsSkipped;
+ public int DbInserted;
+ public int Duplicates;
+ public int Failed;
+ public bool LastSyncUpdated;
+ public string CleanupResult = "SKIPPED";
+ public readonly System.Collections.Generic.List<(string Ip, string EmpNo, DateTime Time)> InsertedPunches =
+ new System.Collections.Generic.List<(string, string, DateTime)>();
+}