Fix crash on torrent filenames in non-UTF-8 encoding (#4144)

Transmission.xcodeproj/project.pbxproj

@@ -396,6 +396,7 @@
 		C3D9062727B7E3E800EF2386 /* psl.c in Sources */ = {isa = PBXBuildFile; fileRef = C3D9061827B7E1DE00EF2386 /* psl.c */; };
 		C3D9062A27B7EAC600EF2386 /* libpsl.h in Headers */ = {isa = PBXBuildFile; fileRef = C3D9061B27B7E31100EF2386 /* libpsl.h */; };
 		C3D9062F27B7F7E200EF2386 /* libpsl.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C3D9062127B7E3C900EF2386 /* libpsl.a */; };
+		C809AEE7291ECFD000BFDBE1 /* NSDataAdditions.mm in Sources */ = {isa = PBXBuildFile; fileRef = C809AEE6291ECFD000BFDBE1 /* NSDataAdditions.mm */; };
 		C841A28129197724009F18E8 /* NSKeyedUnarchiverAdditions.mm in Sources */ = {isa = PBXBuildFile; fileRef = C841A28029197724009F18E8 /* NSKeyedUnarchiverAdditions.mm */; };
 		C86BCD9928228A9600F45599 /* SparkleProxy.mm in Sources */ = {isa = PBXBuildFile; fileRef = C86BCD9828228A9600F45599 /* SparkleProxy.mm */; };
 		C88771AD2803EE7B005C7523 /* libz.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = C88771A92803EE42005C7523 /* libz.tbd */; };
@@ -1176,6 +1177,8 @@
 		C3D9061827B7E1DE00EF2386 /* psl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = psl.c; path = src/psl.c; sourceTree = "<group>"; };
 		C3D9061B27B7E31100EF2386 /* libpsl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = libpsl.h; path = include/libpsl.h; sourceTree = "<group>"; };
 		C3D9062127B7E3C900EF2386 /* libpsl.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libpsl.a; sourceTree = BUILT_PRODUCTS_DIR; };
+		C809AEE5291ECFD000BFDBE1 /* NSDataAdditions.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NSDataAdditions.h; sourceTree = "<group>"; };
+		C809AEE6291ECFD000BFDBE1 /* NSDataAdditions.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = NSDataAdditions.mm; sourceTree = "<group>"; };
 		C81E411127F5BABD00652F56 /* CocoaCompatibility.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CocoaCompatibility.h; sourceTree = "<group>"; };
 		C841A27F29197724009F18E8 /* NSKeyedUnarchiverAdditions.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NSKeyedUnarchiverAdditions.h; sourceTree = "<group>"; };
 		C841A28029197724009F18E8 /* NSKeyedUnarchiverAdditions.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = NSKeyedUnarchiverAdditions.mm; sourceTree = "<group>"; };
@@ -1807,6 +1810,8 @@
 			children = (
 				A29D84021049C25600D1987A /* NSApplicationAdditions.h */,
 				A29D84031049C25600D1987A /* NSApplicationAdditions.mm */,
+				C809AEE5291ECFD000BFDBE1 /* NSDataAdditions.h */,
+				C809AEE6291ECFD000BFDBE1 /* NSDataAdditions.mm */,
 				A234EA521453563B000F3E97 /* NSImageAdditions.h */,
 				A234EA531453563B000F3E97 /* NSImageAdditions.mm */,
 				C841A27F29197724009F18E8 /* NSKeyedUnarchiverAdditions.h */,
@@ -3027,6 +3032,7 @@
 				A2725D5D0DE7507C003445E7 /* TrackerTableView.mm in Sources */,
 				A28F4F770E085BDC003A3882 /* ColorTextField.mm in Sources */,
 				A27F0F330E19AD9800B2DB97 /* TorrentGroup.mm in Sources */,
+				C809AEE7291ECFD000BFDBE1 /* NSDataAdditions.mm in Sources */,
 				A222E9870E6B21D9009FB003 /* BlocklistDownloaderViewController.mm in Sources */,
 				A222EA7B0E6C32C4009FB003 /* BlocklistScheduler.mm in Sources */,
 				A232F07E0EEA034A00041646 /* BonjourController.mm in Sources */,

macosx/CMakeLists.txt

@@ -71,6 +71,7 @@ set(${PROJECT_NAME}_SOURCES
     main.mm
     MessageWindowController.mm
     NSApplicationAdditions.mm
+    NSDataAdditions.mm
     NSImageAdditions.mm
     NSKeyedUnarchiverAdditions.mm
     NSMutableArrayAdditions.mm
@@ -143,6 +144,7 @@ set(${PROJECT_NAME}_HEADERS
     InfoWindowController.h
     MessageWindowController.h
     NSApplicationAdditions.h
+    NSDataAdditions.h
     NSImageAdditions.h
     NSKeyedUnarchiverAdditions.h
     NSMutableArrayAdditions.h

macosx/NSDataAdditions.h

@@ -0,0 +1,15 @@
+// This file Copyright © 2022 Transmission authors and contributors.
+// It may be used under the MIT (SPDX: MIT) license.
+// License text can be found in the licenses/ folder.
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface NSData (NSDataAdditions)
+
+- (NSString*)hexString;
+
+@end
+
+NS_ASSUME_NONNULL_END

macosx/NSDataAdditions.mm

@@ -0,0 +1,33 @@
+// This file Copyright © 2022 Transmission authors and contributors.
+// It may be used under the MIT (SPDX: MIT) license.
+// License text can be found in the licenses/ folder.
+
+#import "NSDataAdditions.h"
+
+@implementation NSData (Additions)
+
+// hexChars from Peter, Aug 19 '14: https://stackoverflow.com/a/25378464
+- (NSString*)hexString
+{
+    char const* hexChars = "0123456789ABCDEF";
+    NSUInteger length = self.length;
+    unsigned char const* bytes = (unsigned char const*)self.bytes;
+    char* chars = (char*)malloc(length * 2);
+    if (chars == NULL)
+    {
+        // malloc returns null if attempting to allocate more memory than the system can provide. Thanks Cœur
+        [NSException raise:@"NSInternalInconsistencyException" format:@"failed malloc" arguments:nil];
+        return nil;
+    }
+    char* s = chars;
+    NSUInteger i = length;
+    while (i--)
+    {
+        *s++ = hexChars[*bytes >> 4];
+        *s++ = hexChars[*bytes & 0xF];
+        bytes++;
+    }
+    return [[NSString alloc] initWithBytesNoCopy:chars length:length * 2 encoding:NSASCIIStringEncoding freeWhenDone:YES];
+}
+
+@end

macosx/Torrent.mm

@@ -16,6 +16,7 @@
 #import "Torrent.h"
 #import "GroupsController.h"
 #import "FileListNode.h"
+#import "NSDataAdditions.h"
 #import "NSStringAdditions.h"
 #import "TrackerNode.h"
 
@@ -1793,8 +1794,26 @@ bool trashDataFile(char const* filename, void* /*user_data*/, tr_error** error)
         {
             auto const file = tr_torrentFile(self.fHandle, i);
 
+            // UTF-8 encoding
             NSString* fullPath = @(file.name);
+            if (!fullPath)
+            {
+                // autodetection of the encoding (#3434)
+                NSData* data = [NSData dataWithBytes:(void const*)file.name length:sizeof(unsigned char) * strlen(file.name)];
+                [NSString stringEncodingForData:data encodingOptions:nil convertedString:&fullPath usedLossyConversion:nil];
+                if (!fullPath)
+                {
+                    // hexa encoding
+                    fullPath = data.hexString;
+                }
+            }
             NSArray* pathComponents = fullPath.pathComponents;
+            while (pathComponents.count <= 1)
+            {
+                // file.name isn't a path: append an arbitrary empty component until we have two components.
+                // Invalid filenames and duplicate filenames don't need to be handled here.
+                pathComponents = [pathComponents arrayByAddingObject:@""];
+            }
 
             if (!tempNode)
             {