[RFC PATCH] git-p4: add option to store files in Git LFS on import

[larsxschneider@gmail.com]

From: Lars Schneider <larsxschneider@gmail.com>

Signed-off-by: Lars Schneider <larsxschneider@gmail.com>
---
 Documentation/git-p4.txt |  12 ++
 git-p4.py                |  94 ++++++++++++++--
 t/t9822-git-p4-lfs.sh    | 277 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 374 insertions(+), 9 deletions(-)
 create mode 100755 t/t9822-git-p4-lfs.sh

diff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt
index 82aa5d6..a188840 100644
--- a/Documentation/git-p4.txt
+++ b/Documentation/git-p4.txt
@@ -252,6 +252,18 @@ Git repository:
 	Use a client spec to find the list of interesting files in p4.
 	See the "CLIENT SPEC" section below.
 
+--use-lfs-if-size-exceeds <n>::
+	Store files that have an uncompressed size exceeding 'n' bytes in 
+	Git LFS. Download and install the Git LFS command line extension to
+	use that option.
+	More info here: https://git-lfs.github.com/
+
+--use-lfs-for-extension <extension>::
+	Store files with 'extension' in Git LFS. Do not prefix the extensions
+	with a '.'. You can use this option multiple times. Download and 
+	install the Git LFS command line extension to use that option.
+	More info here: https://git-lfs.github.com/
+
 -/ <path>::
 	Exclude selected depot paths when cloning or syncing.
 
diff --git a/git-p4.py b/git-p4.py
index 073f87b..e031021 100755
--- a/git-p4.py
+++ b/git-p4.py
@@ -22,6 +22,7 @@ import platform
 import re
 import shutil
 import stat
+import errno
 
 try:
     from subprocess import CalledProcessError
@@ -104,6 +105,16 @@ def chdir(path, is_client_path=False):
         path = os.getcwd()
     os.environ['PWD'] = path
 
+def mkdir_p(path):
+    # Copied from http://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
+    try:
+        os.makedirs(path)
+    except OSError as exc: # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
 def die(msg):
     if verbose:
         raise Exception(msg)
@@ -1994,6 +2005,11 @@ class P4Sync(Command, P4UserMap):
                 optparse.make_option("-/", dest="cloneExclude",
                                      action="append", type="string",
                                      help="exclude depot path"),
+                optparse.make_option("--use-lfs-if-size-exceeds", dest="lfsMinimumFileSize", type="int",
+                                     help="Use LFS to store files bigger than the given threshold in bytes."),
+                optparse.make_option("--use-lfs-for-extension", dest="lfsFileExtensions",
+                                     action="append", type="string",
+                                     help="Use LFS to store files with the given file extension(s)."),
         ]
         self.description = """Imports from Perforce into a git repository.\n
     example:
@@ -2025,6 +2041,9 @@ class P4Sync(Command, P4UserMap):
         self.clientSpecDirs = None
         self.tempBranches = []
         self.tempBranchLocation = "git-p4-tmp"
+        self.lfsFiles = []
+        self.lfsMinimumFileSize = None
+        self.lfsFileExtensions = []
 
         if gitConfig("git-p4.syncFromOrigin") == "false":
             self.syncWithOrigin = False
@@ -2145,6 +2164,63 @@ class P4Sync(Command, P4UserMap):
 
         return branches
 
+    def writeToGitStream(self, gitMode, relPath, contents):
+        self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
+        self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
+        for d in contents:
+            self.gitStream.write(d)
+        self.gitStream.write('\n')
+
+    def writeGitAttributesToStream(self):
+        gitAttributes = [f + ' filter=lfs -text\n' for f in self.lfsFiles if not self.hasFileLFSExtension(f)]
+        self.writeToGitStream(
+            '100644',
+            '.gitattributes',
+            ['*.' + f + ' filter=lfs -text\n' for f in self.lfsFileExtensions] +
+            [f + ' filter=lfs -text\n' for f in self.lfsFiles if not self.hasFileLFSExtension(f)]
+        )
+
+    def hasFileLFSExtension(self, relPath):
+        return reduce(
+            lambda a, b: a or b,
+            [relPath.endswith('.' + e) for e in self.lfsFileExtensions],
+            False
+        )
+
+    def isFileLargerThanLFSTreshold(self, relPath, contents):
+        return self.lfsMinimumFileSize and sum(len(d) for d in contents) >= self.lfsMinimumFileSize
+
+    def generateLFSPointerFile(self, relPath, contents):
+        # Write P4 content to temp file
+        p4ContentTempFile = tempfile.NamedTemporaryFile(prefix='git-lfs', delete=False)
+        for d in contents:
+            p4ContentTempFile.write(d)
+        p4ContentTempFile.flush()
+
+        # Generate LFS pointer file based on P4 content
+        lfsProcess = subprocess.Popen(
+            ['git', 'lfs', 'pointer', '--file=' + p4ContentTempFile.name],
+            stdout=subprocess.PIPE
+        )
+        lfsPointerFile = lfsProcess.stdout.read()
+        if lfsProcess.wait():
+            die('git-lfs command failed. Did you install the extension?')
+        contents = [i+'\n' for i in lfsPointerFile.split('\n')[2:][:-1]]
+
+        # Write P4 content to LFS
+        oid = contents[1].split(' ')[1].split(':')[1][:-1]
+        oidPath = os.path.join(self.cloneDestination, '.git', 'lfs', 'objects', oid[:2], oid[2:4])
+        mkdir_p(oidPath)
+        shutil.move(p4ContentTempFile.name, os.path.join(oidPath, oid))
+
+        # Update Git attributes
+        self.lfsFiles.append(relPath)
+        self.writeGitAttributesToStream()
+
+        # LFS Spec states that pointer files should not have the executable bit set.
+        gitMode = '100644'
+        return (gitMode, contents)
+
     # output one file from the P4 stream
     # - helper for streamP4Files
 
@@ -2213,17 +2289,13 @@ class P4Sync(Command, P4UserMap):
             text = regexp.sub(r'$\1$', text)
             contents = [ text ]
 
-        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
+        if relPath == '.gitattributes':
+            die('.gitattributes already exists in P4.')
 
-        # total length...
-        length = 0
-        for d in contents:
-            length = length + len(d)
+        if self.isFileLargerThanLFSTreshold(relPath, contents) or self.hasFileLFSExtension(relPath):
+            (git_mode, contents) = self.generateLFSPointerFile(relPath, contents)
 
-        self.gitStream.write("data %d\n" % length)
-        for d in contents:
-            self.gitStream.write(d)
-        self.gitStream.write("\n")
+        self.writeToGitStream(git_mode, relPath, contents)
 
     def streamOneP4Deletion(self, file):
         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
@@ -2231,6 +2303,10 @@ class P4Sync(Command, P4UserMap):
             sys.stderr.write("delete %s\n" % relPath)
         self.gitStream.write("D %s\n" % relPath)
 
+        if relPath in self.lfsFiles:
+            self.lfsFiles.remove(relPath)
+            self.writeGitAttributesToStream()
+
     # handle another chunk of streaming data
     def streamP4FilesCb(self, marshalled):
 
diff --git a/t/t9822-git-p4-lfs.sh b/t/t9822-git-p4-lfs.sh
new file mode 100755
index 0000000..b27bf29
--- /dev/null
+++ b/t/t9822-git-p4-lfs.sh
@@ -0,0 +1,277 @@
+#!/bin/sh
+
+test_description='Clone repositories and store files in LFS'
+
+( git lfs help ) >/dev/null 2>&1 || {
+	skip_all='skipping git p4 LFS tests; no git lfs'
+	test_done
+}
+
+. ./lib-git-p4.sh
+
+test_expect_success 'start p4d' '
+	start_p4d
+'
+
+test_expect_success 'Create repo with binary files' '
+	client_view "//depot/... //client/..." &&
+	(
+		cd "$cli" &&
+		echo "text" >file.txt &&
+		echo "bin 13 bytes" >file.dat &&
+		p4 add file.txt &&
+		p4 add file.dat &&
+		p4 submit -d "Add text and binary file" &&
+		echo "bin 13 bytes" >file2.bin &&
+		p4 add file2.bin &&
+		p4 submit -d "Add another binary file with same content"
+		echo "bin 14 bytess" >file3.bin &&
+		p4 add file3.bin &&
+		p4 submit -d "Add another binary file with different content"
+	)
+'
+
+test_expect_success 'Store files in LFS based on size (10 bytes)' '
+	client_view "//depot/... //client/..." &&
+	git p4 clone --use-client-spec --use-lfs-if-size-exceeds=10 --destination="$git" //depot@all &&
+	test_when_finished cleanup_git &&
+	(
+		cd "$git" &&
+
+		cat >expect <<-\EOF &&
+		.git/lfs/objects/d4/43/d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		.git/lfs/objects/e5/fe/e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		EOF
+		find ".git/lfs/objects" -type f >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file.dat >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file2.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		size 14
+		EOF
+		cat file3.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		file.dat filter=lfs -text
+		file2.bin filter=lfs -text
+		file3.bin filter=lfs -text
+		EOF
+		cat .gitattributes >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'Store files in LFS based on size (14 bytes)' '
+	client_view "//depot/... //client/..." &&
+	git p4 clone --use-client-spec --use-lfs-if-size-exceeds=14 --destination="$git" //depot@all &&
+	test_when_finished cleanup_git &&
+	(
+		cd "$git" &&
+
+		cat >expect <<-\EOF &&
+		.git/lfs/objects/e5/fe/e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		EOF
+		find ".git/lfs/objects" -type f >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		bin 13 bytes
+		EOF
+		cat file.dat >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		bin 13 bytes
+		EOF
+		cat file2.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		size 14
+		EOF
+		cat file3.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		file3.bin filter=lfs -text
+		EOF
+		cat .gitattributes >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'Store files in LFS based on extension (dat)' '
+	client_view "//depot/... //client/..." &&
+	git p4 clone --use-client-spec --use-lfs-for-extension=dat --destination="$git" //depot@all &&
+	test_when_finished cleanup_git &&
+	(
+		cd "$git" &&
+
+		cat >expect <<-\EOF &&
+		.git/lfs/objects/d4/43/d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		EOF
+		find ".git/lfs/objects" -type f >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file.dat >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		bin 13 bytes
+		EOF
+		cat file2.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		bin 14 bytess
+		EOF
+		cat file3.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		*.dat filter=lfs -text
+		EOF
+		cat .gitattributes >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'Store files in LFS based on size (14 bytes) and extension (dat)' '
+	client_view "//depot/... //client/..." &&
+	git p4 clone \
+		--use-client-spec \
+		--use-lfs-if-size-exceeds=14 \
+		--use-lfs-for-extension=dat \
+		--destination="$git" //depot@all &&
+	test_when_finished cleanup_git &&
+	(
+		cd "$git" &&
+
+		cat >expect <<-\EOF &&
+		.git/lfs/objects/d4/43/d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		.git/lfs/objects/e5/fe/e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		EOF
+		find ".git/lfs/objects" -type f >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file.dat >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		bin 13 bytes
+		EOF
+		cat file2.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		size 14
+		EOF
+		cat file3.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		*.dat filter=lfs -text
+		file3.bin filter=lfs -text
+		EOF
+		cat .gitattributes >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'Remove file from repo and store files in LFS based on size (10 bytes)' '
+	client_view "//depot/... //client/..." &&
+	(
+		cd "$cli" &&
+		p4 delete file3.bin &&
+		p4 submit -d "Remove file"
+	) &&
+
+	git p4 clone --use-client-spec --use-lfs-if-size-exceeds=10 --destination="$git" //depot@all &&
+	test_when_finished cleanup_git &&
+	(
+		cd "$git" &&
+
+		# Note that file3 remains here as it referenced in the history
+		cat >expect <<-\EOF &&
+		.git/lfs/objects/d4/43/d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		.git/lfs/objects/e5/fe/e5fec48503cd7b85eb9ffaea3311cde2fe9542078b9640369032b26bb5403fff
+		EOF
+		find ".git/lfs/objects" -type f >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file.dat >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		version https://git-lfs.github.com/spec/v1
+		oid sha256:d443795c1aa3ff7e62afd89d6a86bb84ceba0305f6c22151aa8ee95077a39101
+		size 13
+		EOF
+		cat file2.bin >actual &&
+		test_cmp expect actual &&
+
+		cat >expect <<-\EOF &&
+		file.dat filter=lfs -text
+		file2.bin filter=lfs -text
+		EOF
+		cat .gitattributes >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'Clone repo with existing .gitattributes file' '
+	client_view "//depot/... //client/..." &&
+	(
+		cd "$cli" &&
+
+		echo "*.txt text" >.gitattributes &&
+		p4 add .gitattributes &&
+		p4 submit -d "Add .gitattributes"
+	) &&
+
+	test_must_fail git p4 clone --use-client-spec --destination="$git" //depot 2>errs &&
+	grep ".gitattributes already exists in P4." errs
+'
+
+test_expect_success 'kill p4d' '
+	kill_p4d
+'
+
+test_done
-- 
1.9.5 (Apple Git-50.3)