faster file list ingestion for flat lists

src/main/java/edu/vanderbilt/accre/laurelin/Root.java

@@ -253,10 +253,7 @@ public TTreeDataSourceV2Reader(DataSourceOptions options, CacheFactory basketCac
             logger.trace("construct ttreedatasourcev2reader");
             this.sparkContext = sparkContext;
             try {
-                this.paths = new LinkedList<String>();
-                for (String path: options.paths()) {
-                    this.paths.addAll(IOFactory.expandPathToList(path));
-                }
+                this.paths = (LinkedList<String>)IOFactory.expandPathsToList(options.paths());
                 // FIXME - More than one file, please
                 currFile = TFile.getFromFile(fileCache.getROOTFile(this.paths.get(0)));
                 treeName = options.get("tree").orElse("Events");

src/main/java/edu/vanderbilt/accre/laurelin/root_proxy/HadoopFile.java

@@ -11,11 +11,13 @@
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Arrays;
 import java.util.concurrent.Future;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
@@ -71,6 +73,25 @@ public long getLimit() throws IOException {
         return limit;
     }
 
+    public static List<String> expandPathsToList(String[] paths) throws IOException {
+	LinkedList<String> out = new LinkedList<String>();
+	Configuration conf = new Configuration();
+	URI uri = URI.create(paths[0]);
+	FileSystem fileSystem = FileSystem.get(uri, conf);
+	Path[] hpaths = new Path[paths.length];
+	for(int i = 0; i < paths.length; ++i) { hpaths[i] = new Path(paths[i]); }
+	FileStatus[] statoos  = fileSystem.listStatus(hpaths);
+	for (int i = 0; i < statoos.length; ++i) {
+	    String strpath = statoos[i].getPath().toString();
+	    if((statoos[i].isFile() || statoos[i].isSymlink()) && strpath.endsWith(".root")) {
+		out.add(strpath);
+	    } else if(statoos[i].isDirectory()) {
+		out.addAll(HadoopFile.expandPathToList(strpath));
+	    }
+	}
+	return out;
+    }
+
     public static List<String> expandPathToList(String path) throws IOException {
         Configuration conf = new Configuration();
         URI uri = URI.create(path);

src/main/java/edu/vanderbilt/accre/laurelin/root_proxy/IOFactory.java

@@ -36,6 +36,14 @@ public static FileInterface openForRead(String path) throws IOException {
         return ret;
     }
 
+    public static List<String> expandPathsToList(String[] paths) throws IOException {
+        if (Pattern.matches(hadoopPattern,  paths[0])) {
+            return HadoopFile.expandPathsToList(paths);
+        } else {
+            return NIOFile.expandPathsToList(paths);
+        }
+    }
+
     public static List<String> expandPathToList(String path) throws IOException {
         if (Pattern.matches(hadoopPattern,  path)) {
             return HadoopFile.expandPathToList(path);

src/main/java/edu/vanderbilt/accre/laurelin/root_proxy/NIOFile.java

@@ -15,11 +15,13 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+
 public class NIOFile implements FileInterface {
     private RandomAccessFile fh;
     private FileChannel channel;
@@ -68,6 +70,14 @@ public long getLimit() throws IOException {
         return fh.length();
     }
 
+    public static List<String> expandPathsToList(String[] paths) throws IOException {
+	LinkedList<String> out = new LinkedList<String>();
+        for(int i = 0; i < paths.length; ++i) {
+	    out.addAll(NIOFile.expandPathToList(paths[i]));
+	}
+        return out;
+    }
+
     public static List<String> expandPathToList(String path) throws IOException {
         File tmp = FileSystems.getDefault().getPath(path).toFile();
         if (!tmp.isDirectory()) {

src/test/java/edu/vanderbilt/accre/root_proxy/IOTest.java

@@ -13,6 +13,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
+import java.util.LinkedList;
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -171,4 +172,22 @@ public void searchDirectory_hadoop() throws IOException {
         assertEquals(3, files.size());
     }
 
+    @Test
+    public void searchDirectory_nio_paths() throws IOException {
+	String[] paths = new String[1];
+	paths[0] = "testdata/recursive";
+        List<String> files = IOFactory.expandPathsToList(paths);
+        assertEquals(3, files.size());
+    }
+
+    @Test
+    public void searchDirectory_hadoop_paths() throws IOException {
+        Path currentRelativePath = Paths.get("");
+        String s = currentRelativePath.toAbsolutePath().toString();
+	String[] paths = new String[1];
+	paths[0] = "file:///" + s + "/" + "testdata/recursive";
+        List<String> files = IOFactory.expandPathsToList(paths);
+        assertEquals(3, files.size());
+    }
+
 }