feat: enrich initial departmental sync with site-aware failure reporting

Improve initial/departmental sync logging: include location_site_id on failures, distinguish already-present vs added, and resolve targets via the new DB-based target collection path.

HikvisionAttendanceManager.InitialSync.cs

@@ -50,14 +50,25 @@ internal sealed partial class HikvisionAttendanceManager
             .ToList();
         var allActiveDepartmentsMode = deptIds.Count == 0;
 
+        var configuredTargetIps = (_config.TargetMachineIps ?? new List<string>())
+            .Where(x => !string.IsNullOrWhiteSpace(x))
+            .Select(x => x.Trim())
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
         var targets = ResolveInitialSyncTargets();
         if (targets.Count == 0)
         {
-            _logger.OpsWarn("INITIAL_SYNC", "skipped — no valid TargetMachineIps");
+            var reason = configuredTargetIps.Count == 0
+                ? "No target machines are configured. Set TargetMachineIps in DeviceSettings.config."
+                : "Target machines are configured (" + string.Join(", ", configuredTargetIps) +
+                  ") but none passed ISAPI connectivity on HTTP port " +
+                  (_config.IsapiHttpPort > 0 ? _config.IsapiHttpPort : 80) +
+                  ". Machine was found in DB but Connection Failed — check network/firewall, device web port, and credentials.";
+            _logger.OpsWarn("INITIAL_SYNC", "skipped — " + reason);
             _logger.Biz(BizChannel.DepartmentalSync,
                 "DEPARTMENTAL SYNC FAILED",
                 "",
-                "Reason : No valid target machines are configured.",
+                "Reason : " + reason,
                 "");
             _logger.BizSeparator(BizChannel.DepartmentalSync);
             return;
@@ -105,28 +116,6 @@ internal sealed partial class HikvisionAttendanceManager
         foreach (var target in targets)
         {
             ct.ThrowIfCancellationRequested();
-            var hasSession = _sessions.Any(s =>
-                string.Equals((s.Device.Ip ?? "").Trim(), (target.Ip ?? "").Trim(), StringComparison.OrdinalIgnoreCase) ||
-                DeviceIdentity.CanonicalLookupKey(s.Device.DeviceId) == DeviceIdentity.CanonicalLookupKey(target.DeviceId));
-            if (!hasSession)
-            {
-                _logger.OpsWarn("INITIAL_SYNC",
-                    "target=" + (target.DeviceId ?? "") + " ip=" + (target.Ip ?? "") +
-                    " SKIPPED reason=\"target offline\"");
-                _logger.Biz(BizChannel.UserSync,
-                    "MACHINE " + (target.DeviceId ?? ""),
-                    "",
-                    "Machine is not connected.",
-                    "");
-                _logger.BizSeparator(BizChannel.UserSync);
-                WriteDepartmentalSyncSummary(
-                    target,
-                    selection,
-                    employeeIdFilter,
-                    false,
-                    Array.Empty<InitialSyncEmployeeOutcome>());
-                continue;
-            }
 
             _logger.Biz(BizChannel.UserSync,
                 "MACHINE " + (target.DeviceId ?? "") + " -> Total Users : " + employees.Count,
@@ -157,12 +146,20 @@ internal sealed partial class HikvisionAttendanceManager
                 var photoUrl = "";
                 var photoDownloaded = false;
                 var userCreated = false;
+                var userAlreadyPresent = false;
                 var faceUploaded = false;
                 var reason = "";
 
                 try
                 {
-                    if (!existingNos.Contains(employeeNo))
+                    userAlreadyPresent = existingNos.Contains(employeeNo);
+                    if (!userAlreadyPresent && VerifyUserOnTarget(target, employeeNo, out _))
+                    {
+                        userAlreadyPresent = true;
+                        existingNos.Add(employeeNo);
+                    }
+
+                    if (!userAlreadyPresent)
                     {
                         var dto = new UserDto
                         {
@@ -171,12 +168,19 @@ internal sealed partial class HikvisionAttendanceManager
                             NumOfFace = 0,
                             NumOfFp = 0
                         };
-                        if (!CreateUserOnTarget(target, dto, maxRetries, ct, out var createErr))
+                        if (!CreateUserOnTarget(target, dto, maxRetries, ct, out var createErr, out var alreadyExisted))
                         {
                             reason = string.IsNullOrWhiteSpace(createErr) ? "user_create_failed" : createErr;
                             continue;
                         }
 
+                        if (alreadyExisted)
+                        {
+                            userAlreadyPresent = true;
+                            existingNos.Add(employeeNo);
+                        }
+                        else
+                        {
                             userCreated = true;
                             existingNos.Add(employeeNo);
                             if (_config.EnableDbIntegration && _attendanceMachineUserRepository != null)
@@ -189,6 +193,7 @@ internal sealed partial class HikvisionAttendanceManager
                                     out _);
                             }
                         }
+                    }
 
                     if (!_config.EnableEmployeePhotoSource || string.IsNullOrWhiteSpace(photoBase))
                     {
@@ -238,18 +243,27 @@ internal sealed partial class HikvisionAttendanceManager
                     {
                         DepartmentId = emp.DepartmentId,
                         EmployeeNo = employeeNo,
+                        LocationSiteId = emp.LocationSiteId,
                         Synced = faceUploaded,
                         Reason = faceUploaded
                             ? ""
                             : (string.IsNullOrWhiteSpace(reason) ? "face_upload_failed" : reason)
                     });
-                    WriteInitialSyncEmployeeBizLog(target, employeeNo, userCreated, faceUploaded, reason);
+                    WriteInitialSyncEmployeeBizLog(
+                        target,
+                        employeeNo,
+                        emp.LocationSiteId,
+                        userCreated,
+                        userAlreadyPresent,
+                        faceUploaded,
+                        reason);
 
                     if (photoDownloaded && faceUploaded)
                     {
                         _logger.Ops("INITIAL_SYNC",
                             "employeeNo=" + employeeNo +
                             " employeeId=" + employeeId +
+                            " locationSiteId=" + emp.LocationSiteId +
                             " photoUrl=" + photoUrl +
                             " photoDownloaded=true" +
                             " userCreated=" + (userCreated ? "true" : "false") +
@@ -260,6 +274,7 @@ internal sealed partial class HikvisionAttendanceManager
                         _logger.Ops("INITIAL_SYNC",
                             "employeeNo=" + employeeNo +
                             (string.IsNullOrWhiteSpace(employeeId) ? "" : (" employeeId=" + employeeId)) +
+                            " locationSiteId=" + emp.LocationSiteId +
                             (string.IsNullOrWhiteSpace(photoUrl) ? "" : (" photoUrl=" + photoUrl)) +
                             " photoDownloaded=false" +
                             " reason=" + (string.IsNullOrWhiteSpace(reason) ? "photo_not_found" : reason));
@@ -269,6 +284,7 @@ internal sealed partial class HikvisionAttendanceManager
                         _logger.Ops("INITIAL_SYNC",
                             "employeeNo=" + employeeNo +
                             " employeeId=" + employeeId +
+                            " locationSiteId=" + emp.LocationSiteId +
                             " photoUrl=" + photoUrl +
                             " photoDownloaded=true" +
                             " userCreated=" + (userCreated ? "true" : "false") +
@@ -291,19 +307,23 @@ internal sealed partial class HikvisionAttendanceManager
     private void WriteInitialSyncEmployeeBizLog(
         HikvisionAttendanceWindowsService.DeviceConfig target,
         string employeeNo,
+        int locationSiteId,
         bool userCreated,
+        bool userAlreadyPresent,
         bool faceUploaded,
         string reason)
     {
         if (userCreated)
         {
             _logger.Totals.UsersAdded++;
-            _logger.Biz(BizChannel.UserSync,
+            _logger.Biz(BizChannel.UserSync, employeeNo + " added successfully.", "");
-                "Machine ID : " + (target.DeviceId ?? ""),
+            _logger.Biz(BizChannel.DepartmentalSync, employeeNo + " added successfully.", "");
-                "Machine IP : " + (target.Ip ?? ""),
+        }
-                "",
+        else if (userAlreadyPresent)
-                employeeNo + " added successfully.",
+        {
-                "");
+            // Print this line only when the user was confirmed present on the device.
+            _logger.Biz(BizChannel.UserSync, employeeNo + " already present.", "");
+            _logger.Biz(BizChannel.DepartmentalSync, employeeNo + " already present.", "");
         }
 
         if (faceUploaded)
@@ -313,14 +333,70 @@ internal sealed partial class HikvisionAttendanceManager
             return;
         }
 
+        if (string.IsNullOrWhiteSpace(reason))
+            return;
+
         _logger.Totals.TemplatesFailed++;
+        var failureTitle = ClassifyInitialSyncFailureTitle(reason);
+        var siteLine = "HRMS location_site_id : " + locationSiteId;
         _logger.Biz(BizChannel.UserSync,
-            employeeNo + " face enrollment failed.",
+            employeeNo + " " + failureTitle,
+            "",
+            siteLine,
             "",
             "Reason :",
             "",
             DescribeInitialSyncFailure(reason),
             "");
+        _logger.Biz(BizChannel.DepartmentalSync,
+            "Employee " + employeeNo + " failed.",
+            "",
+            siteLine,
+            "",
+            "Reason : " + DescribeInitialSyncFailure(reason),
+            "");
+    }
+
+    private static string ClassifyInitialSyncFailureTitle(string reason)
+    {
+        if (string.IsNullOrWhiteSpace(reason))
+            return "sync failed.";
+
+        switch (reason.Trim())
+        {
+            case "user_create_failed":
+                return "user create failed.";
+            case "photo_source_disabled":
+            case "missing_employee_id":
+            case "photo_not_found":
+            case "invalid_jpeg":
+                return "photo sync failed.";
+            case "FACE_NOT_DETECTED":
+            case "face_upload_failed":
+                return "face enrollment failed.";
+        }
+
+        if (reason.IndexOf("401", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("Unauthorized", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("Authentication", StringComparison.OrdinalIgnoreCase) >= 0)
+            return "device authentication failed.";
+
+        if (reason.IndexOf("UserInfo/Record", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.StartsWith("ISAPI error", StringComparison.OrdinalIgnoreCase) ||
+            reason.StartsWith("HTTP ", StringComparison.OrdinalIgnoreCase))
+            return "user create failed.";
+
+        if (reason.IndexOf("FACE_NOT_DETECTED", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("FaceDataRecord", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("FDLib", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("face", StringComparison.OrdinalIgnoreCase) >= 0)
+            return "face enrollment failed.";
+
+        if (reason.StartsWith("photo_", StringComparison.OrdinalIgnoreCase) ||
+            reason.IndexOf("photo", StringComparison.OrdinalIgnoreCase) >= 0)
+            return "photo sync failed.";
+
+        return "sync failed.";
     }
 
     private static string DescribeInitialSyncFailure(string reason)
@@ -328,6 +404,14 @@ internal sealed partial class HikvisionAttendanceManager
         if (string.IsNullOrWhiteSpace(reason))
             return "Unknown error.";
 
+        if (reason.IndexOf("401", StringComparison.OrdinalIgnoreCase) >= 0 ||
+            reason.IndexOf("Unauthorized", StringComparison.OrdinalIgnoreCase) >= 0)
+        {
+            return "Device rejected login (HTTP 401 Unauthorized). " +
+                   "Username/password for this target IP are wrong in serviceconfig.json Devices. " +
+                   "This is not related to HRMS location_site_id. Device detail: " + reason.Trim();
+        }
+
         switch (reason)
         {
             case "FACE_NOT_DETECTED":
@@ -342,8 +426,12 @@ internal sealed partial class HikvisionAttendanceManager
                 return "Employee has no HRMS id, photo URL cannot be built.";
             case "user_create_failed":
                 return "User could not be created on the device.";
+            case "face_upload_failed":
+                return "Face upload to the device failed.";
             default:
-                return reason;
+                if (reason.StartsWith("photo_http_", StringComparison.OrdinalIgnoreCase))
+                    return "Employee photo download failed (" + reason + "). Portal may require auth or the URL is blocked.";
+                return reason.Trim();
         }
     }
 
@@ -385,34 +473,8 @@ internal sealed partial class HikvisionAttendanceManager
         return true;
     }
 
-    private List<HikvisionAttendanceWindowsService.DeviceConfig> ResolveInitialSyncTargets()
+    private List<HikvisionAttendanceWindowsService.DeviceConfig> ResolveInitialSyncTargets() =>
-    {
+        CollectTestedSyncTargets(source: null, hasSource: false, CancellationToken.None);
-        var targets = new List<HikvisionAttendanceWindowsService.DeviceConfig>();
-        foreach (var targetIp in _config.TargetMachineIps ?? Enumerable.Empty<string>())
-        {
-            var t = ResolveDeviceConfigByIp(targetIp);
-            if (t == null || string.IsNullOrWhiteSpace(t.Ip))
-            {
-                _logger.OpsWarn("INITIAL_SYNC", "target skipped — not found or IP empty. TargetMachineIp=\"" + targetIp + "\"");
-                continue;
-            }
-            if (targets.Any(x => string.Equals(x.Ip, t.Ip, StringComparison.OrdinalIgnoreCase)))
-                continue;
-            targets.Add(t);
-        }
-
-        foreach (var tid in _config.TargetDeviceIds ?? Enumerable.Empty<string>())
-        {
-            var t = ResolveDeviceConfig(tid);
-            if (t == null || string.IsNullOrWhiteSpace(t.Ip))
-                continue;
-            if (targets.Any(x => string.Equals(x.Ip, t.Ip, StringComparison.OrdinalIgnoreCase)))
-                continue;
-            targets.Add(t);
-        }
-
-        return targets;
-    }
 
     private InitialSyncSelection LoadEmployeesForInitialDepartmentSync(
         IReadOnlyList<string> departmentIds,
@@ -496,7 +558,7 @@ internal sealed partial class HikvisionAttendanceManager
                 return result;
 
             var employeeSql = new StringBuilder(
-                "SELECT e.id, e.serial_number, e.department_id " +
+                "SELECT e.id, e.serial_number, e.department_id, e.location_site_id " +
                 "FROM employee e " +
                 "INNER JOIN department d ON d.id = e.department_id " +
                 "WHERE d.is_active = 1 AND e.is_active = 1 " +
@@ -529,6 +591,7 @@ internal sealed partial class HikvisionAttendanceManager
                         Id = rd["id"]?.ToString()?.Trim() ?? "",
                         SerialNumber = serial,
                         DepartmentId = rd["department_id"]?.ToString()?.Trim() ?? "",
+                        LocationSiteId = ReadNullableInt(rd["location_site_id"]) ?? locationSiteId,
                         Name = ""
                     });
                 }
@@ -666,26 +729,35 @@ internal sealed partial class HikvisionAttendanceManager
             lines.Add("Eligible Users : " + eligibleCount);
             lines.Add("Synced Users : " + syncedCount);
             lines.Add("Failed Users : " + failedCount);
+            if (!string.IsNullOrWhiteSpace(reason))
+                lines.Add("Reason : " + reason);
+
             if (failedCount > 0)
-                lines.Add("Failed Employee Numbers : " +
+            {
-                    string.Join(",", targetOnline
+                lines.Add("Failed Users Detail :");
-                        ? departmentOutcomes.Where(x => !x.Synced).Select(x => x.EmployeeNo)
+                if (targetOnline)
-                        : selection.Employees
+                {
+                    foreach (var fail in departmentOutcomes
+                                 .Where(x => !x.Synced)
+                                 .OrderBy(x => x.EmployeeNo, StringComparer.OrdinalIgnoreCase))
+                    {
+                        lines.Add("  " + FormatFailedEmployeeLine(fail.EmployeeNo, fail.LocationSiteId,
+                            DescribeInitialSyncFailure(fail.Reason)));
+                    }
+                }
+                else
+                {
+                    foreach (var emp in selection.Employees
                                  .Where(x => string.Equals(
                                      x.DepartmentId,
                                      departmentId,
                                      StringComparison.OrdinalIgnoreCase))
-                            .Select(x => x.SerialNumber)));
+                                 .OrderBy(x => x.SerialNumber, StringComparer.OrdinalIgnoreCase))
-            if (!string.IsNullOrWhiteSpace(reason))
-                lines.Add("Reason : " + reason);
-
-            foreach (var failureGroup in departmentOutcomes
-                         .Where(x => !x.Synced)
-                         .GroupBy(x => DescribeInitialSyncFailure(x.Reason))
-                         .OrderBy(x => x.Key, StringComparer.OrdinalIgnoreCase))
                     {
-                lines.Add("Failure Reason : " + failureGroup.Key +
+                        lines.Add("  " + FormatFailedEmployeeLine(emp.SerialNumber, emp.LocationSiteId,
-                    " (" + failureGroup.Count() + " user(s))");
+                            "Target machine is offline."));
+                    }
+                }
             }
 
             lines.Add("");
@@ -722,14 +794,46 @@ internal sealed partial class HikvisionAttendanceManager
         }
 
         var totalSynced = outcomes.Count(x => x.Synced);
+        var totalFailed = Math.Max(0, selection.Employees.Count - totalSynced);
         lines.Add("TOTAL SYNCED USERS : " + totalSynced);
-        lines.Add("TOTAL FAILED USERS : " + Math.Max(0, selection.Employees.Count - totalSynced));
+        lines.Add("TOTAL FAILED USERS : " + totalFailed);
+        if (totalFailed > 0)
+        {
+            lines.Add("");
+            lines.Add("CONCLUSION — FAILED USERS:");
+            foreach (var fail in outcomes
+                         .Where(x => !x.Synced)
+                         .OrderBy(x => x.EmployeeNo, StringComparer.OrdinalIgnoreCase))
+            {
+                lines.Add("  " + FormatFailedEmployeeLine(fail.EmployeeNo, fail.LocationSiteId,
+                    DescribeInitialSyncFailure(fail.Reason)));
+            }
+        }
+
         lines.Add("");
 
         _logger.Biz(BizChannel.DepartmentalSync, lines.ToArray());
+        _logger.Biz(BizChannel.UserSync, lines.ToArray());
         _logger.BizSeparator(BizChannel.DepartmentalSync);
     }
 
+    private static string FormatFailedEmployeeLine(string employeeNo, int locationSiteId, string reason) =>
+        employeeNo + " [HRMS location_site_id=" + locationSiteId + "] : " + reason;
+
+    private static int? ReadNullableInt(object value)
+    {
+        if (value == null || value == DBNull.Value)
+            return null;
+        if (value is int i)
+            return i;
+        if (int.TryParse(Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture)?.Trim(),
+                System.Globalization.NumberStyles.Integer,
+                System.Globalization.CultureInfo.InvariantCulture,
+                out var parsed))
+            return parsed;
+        return null;
+    }
+
     private static void AppendSqlParameters(StringBuilder sql, string prefix, int count)
     {
         for (int i = 0; i < count; i++)
@@ -830,6 +934,7 @@ internal sealed partial class HikvisionAttendanceManager
         public string Id { get; set; } = "";
         public string SerialNumber { get; set; } = "";
         public string DepartmentId { get; set; } = "";
+        public int LocationSiteId { get; set; }
         public string Name { get; set; } = "";
     }
 
@@ -837,6 +942,7 @@ internal sealed partial class HikvisionAttendanceManager
     {
         public string DepartmentId { get; set; } = "";
         public string EmployeeNo { get; set; } = "";
+        public int LocationSiteId { get; set; }
         public bool Synced { get; set; }
         public string Reason { get; set; } = "";
     }