Organize Files into Directories Recursively using Node.js (Advanced)

Ever wondered how you could merge contents of folders based on when the files were created? This post will cover how I used Node.js to take a manual tedious work and automate the work for me.

In my previous post on how I programmed a small files organizer using Node.js, I realized that sometimes I come across other situations where I need to “merge” directories that share the same name but contain different photos.

As you can see from the screenshot above, because Mac OS X doesn’t allow merging directories sharing the same name. So they’ll end up replacing the content of the destination directory altogether, ouch!

Revisiting our Node.js script

If you aren’t familiar with the first iteration of the script, you might want to check my original post here.

In my first iteration of that script.js, I was mostly expecting that all my files/photos will be flat in my source directory, and would end up getting organized according to their creation date.

let root = './photos'
const outputDir = './output'
const fs = require('fs')

console.log('\n------------------------------------')
console.log('🗂  Welcome to KB\'s File Organizer 🗂')
console.log('------------------------------------\n')

// pull the arguments (by skipping the first 2 being the path and the script name)
const args = process.argv.slice(2)
// args.forEach((val, index) => {
//   console.log(`${index}: ${val}`)
// })

// check if we have the proper argument for the source director
// and make sure it does exist.
if (args[0] && fs.existsSync(args[0])) {
  root = args[0]
}

console.log(`> PATH "${root}"   -->   "${outputDir}"`)

fs.readdir(root, (err, files) => {
  if (err) {
    console.log('Error getting directory information: ', JSON.stringify(err, null, 2))
    return
  }

  // remove all hidden files
  files = files.filter(item => !(/(^|\/)\.[^\/\.]/g).test(item))

  // mention how many files total
  const totalFiles = files.length
  console.log(`> TOTAL FILES (${totalFiles})\n`)

  // create the output directory
  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir)
  }

  let i = 0
  // keep track of the files ones
  let errorFiles = []

  files.forEach(file => {
    // increase the current file count
    ++i

    const stats = fs.statSync(`${root}/${file}`)

    // if that's a directory let's skip it.
    if (!stats.isDirectory()) {
      const birthtime = stats.birthtime

      let month = pad(birthtime.getMonth() + 1)
      let day = pad(birthtime.getDate())
      let year = birthtime.getFullYear()

      // check if we don't have the year folder created, to create it
      const yearDir = `${outputDir}/${year}`
      if (!fs.existsSync(yearDir)) {
        fs.mkdirSync(yearDir)
      }
      // same with month
      const monthDir = `${yearDir}/${month}`
      if (!fs.existsSync(monthDir)) {
        fs.mkdirSync(monthDir)
      }
      // same with day
      const dayDir = `${monthDir}/${year}_${month}_${day}`
      if (!fs.existsSync(dayDir)) {
        fs.mkdirSync(dayDir)
      }

      // move the file
      const moveStatus = `@ ${year}/${month}/${day} ${parseInt((i / totalFiles) * 100)}%`
      try {
        fs.renameSync(`${root}/${file}`, `${dayDir}/${file}`)
        console.log('✔️ ', file, moveStatus);
      } catch (err) {
        console.log('❌ ', file, moveStatus, `ERROR: ${err}`);
        errorFiles.push(file)
      }
    }
  });

  console.log(`\n🎉 ALL DONE (${i}/${totalFiles}) 🎉\n`)
  if (errorFiles.length) {
    console.log(`⚠️  ERRORS (${errorFiles.length}/${totalFiles}) ⚠️`)
    errorFiles.forEach((val, index) => {
      console.log(val)
    })
    console.log('\n')
  }
});

function pad(num, size = 2) {
  var s = num + '';
  while (s.length < size) s = '0' + s;
  return s;
}

But with my current scenario that wouldn’t work, that’s because I want to traverse all the folders and subfolders in order to move the files over to an existing/or new folder structure.

Restructuring the script

Here’s the content of the script.js file:

const fs = require('fs')
const organizeFile = require('./organizeFile')

console.log('\n------------------------------------')
console.log('🗂  Welcome to KB\'s File Organizer 🗂')
console.log('------------------------------------\n')


const argv = require('yargs')
  .usage('Usage:')
  .options('from', {
    alias: 'sourceDirectory',
    default: './photos',
    describe: 'The directory containing photos/folders that need to be organized.'
  })
  .options('to', {
    alias: 'destinationDirectory',
    default: './output',
    describe: 'Root directory to store the organized photos.'
  })
  .options('incSub', {
    alias: 'includeSubdirectories',
    default: true,
    describe: 'Whether to organize subdirectories within directories.'
  })
  .help()
  .argv

function traverse (path, outputDir, includeDirectories) {
  fs.readdir(path, (err, files) => {
    if (err) {
      console.log(`Error getting directory information ${path}: ${JSON.stringify(err, null, 2)}`)
      return
    }

    // remove all hidden files
    files = files.filter(item => !(/(^|\/)\.[^\/\.]/g).test(item))

    // mention how many files total
    const totalFiles = files.length
    console.log(`📂 ${path} (${totalFiles})`)

    // create the output directory
    if (!fs.existsSync(outputDir)) {
      fs.mkdirSync(outputDir)
    }

    let i = 0
    // keep track of the files ones
    let errorFiles = []

    files.forEach(file => {
      // increase the current file count
      ++i

      const filePath = `${path}/${file}`

      const stats = fs.statSync(filePath)

      // if that's a directory let's recurse.
      if (stats.isDirectory()) {
        if (includeDirectories === true) {
          traverse(filePath, outputDir, includeDirectories)
        }
      } else {
        // organize that file and see if we got the file back
        const result = organizeFile(filePath, file, outputDir)
        if (result.error) {
          errorFiles.push(result.file)
        }
      }
    })

    console.log(`🗂  ${path} (${i}/${totalFiles}) ✔️\n`)
    if (errorFiles.length) {
      console.log(`⚠️  📂 ERRORS ${path} (${errorFiles.length}/${totalFiles}) ⚠️`)
      errorFiles.forEach((val, index) => {
        console.log(val)
      })
      console.log('\n')
    }
  })
}

traverse(argv.from, argv.to, argv.incSub === 'true')

The first big change is that I introduced yargs as a new npm module, so that I can enhance how the parameters are passed to the script. Where you can do something like:

node script.js --from ./test --to ./test-out --incSub true

Notice how the —-from and —-to can point to a source and destination folders. Don’t forget to run npm i in terminal in the root folder to install that new package!

The other change is that I now check if I have a directory/folder, and if permitted (i.e.: —-incSub true) go ahead and call that same method recursively to visit that folder.

const fs = require('fs')

function pad(num, size = 2) {
  var s = num + '';
  while (s.length < size) s = '0' + s;
  return s;
}

module.exports = function organizeFile (filePath, file, outputDir) {
  //console.log(`📄 ${filePath}`)

  const stats = fs.statSync(filePath)

  // if that's a directory let's skip it.
  if (!stats.isDirectory()) {
    const birthtime = stats.birthtime

    let month = pad(birthtime.getMonth() + 1)
    let day = pad(birthtime.getDate())
    let year = birthtime.getFullYear()

    // check if we don't have the year folder created, to create it
    const yearDir = `${outputDir}/${year}`
    if (!fs.existsSync(yearDir)) {
      fs.mkdirSync(yearDir)
    }
    // same with month
    const monthDir = `${yearDir}/${month}`
    if (!fs.existsSync(monthDir)) {
      fs.mkdirSync(monthDir)
    }
    // same with day
    const dayDir = `${monthDir}/${year}_${month}_${day}`
    if (!fs.existsSync(dayDir)) {
      fs.mkdirSync(dayDir)
    }

    // move the file
    const moveStatus = dayDir
    let error
    try {
      fs.renameSync(filePath, `${dayDir}/${file}`)
      console.log('📄', filePath, `→ ${moveStatus} ✔️`);
    } catch (err) {
      console.log('❌', filePath, moveStatus, `ERROR: ${err}`);
      error = err
    }

    return {
      file,
      error
    }
  }
}

The last change was to pull out all the specific file organization part into its own file and include it at the top organizeFile.js.

That’s pretty much it, now you can run the script and enjoy recursively moving folders to where they belong! 🥳

Usage example to move files non recursively

Notice how that single file was moved to the new destination based on its creation date.

Usage example to move files recursively

Notice how all the files got moved starting with the first one (underneath folder 09), then the other 2 files (underneath 2019_09_28) and finally the 3 files.

I hope you found this post useful, I’ve provided the source code for this project as a zip file below, feel free to download and use it! 🍻

0