Fix file sanitization so that it supports unicode characters.

Replace invalid characters with "_" to make it easier to parse.

MyTracks/src/com/google/android/apps/mytracks/util/FileUtils.java

@@ -16,13 +16,13 @@
 package com.google.android.apps.mytracks.util;
 
 import com.google.android.apps.mytracks.Constants;
+import com.google.common.annotations.VisibleForTesting;
 
 import android.os.Environment;
 
 import java.io.File;
 import java.text.SimpleDateFormat;
 import java.util.TimeZone;
-import java.util.regex.Pattern;
 
 /**
  * Utilities for dealing with files.
@@ -30,16 +30,12 @@ import java.util.regex.Pattern;
  * @author Rodrigo Damazio
  */
 public class FileUtils {
-  /**
-   * The maximum length of a filename, as per the FAT32 specification.
-   */
-  private static final int MAX_FILENAME_LENGTH = 260;
 
   /**
-   * A set of characters that are prohibited from being in file names.
+   * The maximum Fat32 path length.
    */
-  private static final Pattern PROHIBITED_CHAR_PATTERN =
+  @VisibleForTesting
-      Pattern.compile("[^ A-Za-z0-9_.()-]+");
+  static final int MAX_FAT32_PATH_LENGTH = 260;
   
   /**
    * Timestamp format in UTC time zone.
@@ -77,20 +73,6 @@ public class FileUtils {
         Environment.getExternalStorageState());
   }
 
-  /**
-   * Normalizes the input string and make sure it is a valid fat32 file name.
-   *
-   * @param name the name to normalize
-   * @return the sanitized name
-   */
-  String sanitizeName(String name) {
-    String cleaned = PROHIBITED_CHAR_PATTERN.matcher(name).replaceAll("");
-
-    return (cleaned.length() > MAX_FILENAME_LENGTH)
-        ? cleaned.substring(0, MAX_FILENAME_LENGTH)
-        : cleaned.toString();
-  }
-
   /**
    * Ensures the given directory exists by creating it and its parents if
    * necessary.
@@ -125,29 +107,76 @@ public class FileUtils {
   }
 
   /**
-   * Builds a filename with the given base name (prefix) and the given
+   * Builds a filename with the given base and the given extension, possibly
-   * extension, possibly adding a suffix to ensure the file doesn't exist.
+   * adding a suffix to ensure the file doesn't exist.
    *
-   * @param directory the directory the file will live in
+   * @param directory the directory the filename will be located in
-   * @param fileBaseName the prefix for the file name
+   * @param base the base for the filename
-   * @param extension the file's extension
+   * @param extension the extension for the filename
    * @param suffix the first numeric suffix to try to use, or 0 for none
-   * @return the complete file name, without the directory
+   * @return the complete filename, without the directory
    */
-  private String buildUniqueFileName(File directory, String fileBaseName,
+  private String buildUniqueFileName(File directory, String base, String extension, int suffix) {
-      String extension, int suffix) {
+    String suffixName = "";
-    String suffixedBaseName = fileBaseName;
     if (suffix > 0) {
-      suffixedBaseName += " (" + Integer.toString(suffix) + ")";
+      suffixName += "(" + Integer.toString(suffix) + ")";
     }
+    suffixName += "." + extension;
 
-    String fullName = suffixedBaseName + "." + extension;
+    String baseName = sanitizeFileName(base);
-    String sanitizedName = sanitizeName(fullName);
+    baseName = truncateFileName(directory, baseName, suffixName);
-    if (!fileExists(directory, sanitizedName)) {
+    String fullName = baseName + suffixName;
-      return sanitizedName;
+    
+    if (!fileExists(directory, fullName)) {
+      return fullName;
     }
+    return buildUniqueFileName(directory, base, extension, suffix + 1);
+  }
 
-    return buildUniqueFileName(directory, fileBaseName, extension, suffix + 1);
+  /**
+   * Sanitizes the name as a valid fat32 filename. For simplicity, fat32
+   * filename characters may be any combination of letters, digits, or
+   * characters with code point values greater than 127. Replaces the invalid
+   * characters with "_" and collapses multiple "_" together.
+   *
+   * @param name name
+   */
+  @VisibleForTesting
+  String sanitizeFileName(String name) {
+    StringBuffer buffer = new StringBuffer();
+    for (int i = 0; i < name.length(); i++) {
+      int codePoint = name.codePointAt(i);
+      if (codePoint > 127) {
+        buffer.appendCodePoint(codePoint);
+      } else if (Character.isLetterOrDigit(name.charAt(i))) {
+        buffer.appendCodePoint(codePoint);
+      } else {
+        buffer.append("_");
+      }
+    }
+    String result = buffer.toString();
+    return result.replaceAll("_+", "_");
+  }
+
+  /**
+   * Truncates the name if necessary thus the filename path length (directory +
+   * name + suffix) meets the Fat32 path limit.
+   *
+   * @param directory directory
+   * @param name name
+   * @param suffix suffix
+   */
+  @VisibleForTesting
+  String truncateFileName(File directory, String name, String suffix) {
+
+    // 1 at the end accounts for the trailing NUL character
+    int requiredLength = directory.getPath().length() + suffix.length() + 1;
+    if (name.length() + requiredLength > MAX_FAT32_PATH_LENGTH) {
+      int limit = MAX_FAT32_PATH_LENGTH - requiredLength;
+      return name.substring(0, limit);
+    } else {
+      return name;
+    }
   }
 
   /**

MyTracksTest/src/com/google/android/apps/mytracks/util/FileUtilsTest.java

@@ -32,7 +32,7 @@ import junit.framework.TestCase;
  */
 public class FileUtilsTest extends TestCase {
   private static final String ORIGINAL_NAME = "Swim\10ming-^across: the/ pacific (ocean).";
-  private static final String SANITIZED_NAME = "Swimming-across the pacific (ocean).";
+  private static final String SANITIZED_NAME = "Swim_ming_across_the_pacific_ocean_";
 
   private FileUtils fileUtils;
   private Set<String> existingFiles;
@@ -65,31 +65,54 @@ public class FileUtilsTest extends TestCase {
     assertEquals(expectedName, dirName);
   }
 
-  public void testSanitizeName() {
+  public void testSanitizeFileName() {
-    assertEquals(SANITIZED_NAME, fileUtils.sanitizeName(ORIGINAL_NAME));
+    assertEquals(SANITIZED_NAME, fileUtils.sanitizeFileName(ORIGINAL_NAME));
+  }
+
+  public void testTruncateFileName() {
+    File directory = new File("/dir1/dir2/");
+    String suffix = ".gpx";
+    char[] name = new char[FileUtils.MAX_FAT32_PATH_LENGTH];
+    for (int i = 0; i < name.length; i++) {
+      name[i] = 'a';
+    }
+    String nameString = new String(name);
+    
+    int expectedLength = FileUtils.MAX_FAT32_PATH_LENGTH - directory.getPath().length()
+        - suffix.length() - 1;
+    char[] expected = new char[expectedLength];
+    for (int i = 0; i < expected.length; i++) {
+      expected[i] = 'a';
+    }
+    String expectedString = new String(expected);
+    
+    String truncated = fileUtils.truncateFileName(directory, nameString, suffix);
+    assertEquals(expectedString, truncated);
+    assertEquals(FileUtils.MAX_FAT32_PATH_LENGTH,
+        new File(directory, truncated + suffix).getPath().length());
   }
   
   public void testBuildUniqueFileName_someExist() {
     existingFiles = new HashSet<String>();
     existingFiles.add("Filename.ext");
-    existingFiles.add("Filename (1).ext");
+    existingFiles.add("Filename(1).ext");
-    existingFiles.add("Filename (2).ext");
+    existingFiles.add("Filename(2).ext");
-    existingFiles.add("Filename (3).ext");
+    existingFiles.add("Filename(3).ext");
-    existingFiles.add("Filename (4).ext");
+    existingFiles.add("Filename(4).ext");
 
-    String filename = fileUtils.buildUniqueFileName(null, "Filename", "ext");
+    String filename = fileUtils.buildUniqueFileName(new File("/dir/"), "Filename", "ext");
-    assertEquals("Filename (5).ext", filename);
+    assertEquals("Filename(5).ext", filename);
   }
 
   public void testBuildUniqueFileName_oneExists() {
     existingFiles.add("Filename.ext");
 
-    String filename = fileUtils.buildUniqueFileName(null, "Filename", "ext");
+    String filename = fileUtils.buildUniqueFileName(new File("/dir/"), "Filename", "ext");
-    assertEquals("Filename (1).ext", filename);
+    assertEquals("Filename(1).ext", filename);
   }
 
   public void testBuildUniqueFileName_noneExists() {
-    String filename = fileUtils.buildUniqueFileName(null, "Filename", "ext");
+    String filename = fileUtils.buildUniqueFileName(new File("/dir/"), "Filename", "ext");
     assertEquals("Filename.ext", filename);
   }
 }