swr/rast: minimize codegen redundant work

Move filtering of redundant codegen operations into gen scripts themselves

Reviewed-by: Bruce Cherniak <bruce.cherniak@intel.com>

6 files changed, 335 insertions, 128 deletions

diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
index aa09f220c3..c5842aa48d 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_archrast.py
@@ -24,7 +24,7 @@ from __future__ import print_function
 import os
 import sys
 import re
-from gen_common import ArgumentParser, MakoTemplateWriter
+from gen_common import *
 
 def parse_event_fields(lines, idx, event_dict):
     field_names = []
@@ -144,6 +144,10 @@ def main():
         print('Error: Could not find private proto file %s' % proto_private_filename, file=sys.stderr)
         return 1
 
+    final_output_dir = output_dir
+    MakeDir(final_output_dir)
+    output_dir = MakeTmpDir('_codegen')
+
     protos = {}
     protos['events'] = {}       # event dictionary containing events with their fields
     protos['event_names'] = []  # needed to keep events in order parsed. dict is not ordered.
@@ -153,53 +157,64 @@ def main():
     parse_protos(protos, proto_filename)
     parse_protos(protos, proto_private_filename)
 
-    # Generate event header
-    if args.gen_event_hpp:
-        curdir = os.path.dirname(os.path.abspath(__file__))
-        template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.hpp'])
-        output_fullpath = os.sep.join([output_dir, output_filename])
-
-        MakoTemplateWriter.to_file(template_file, output_fullpath,
-                cmdline=sys.argv,
-                filename=output_filename,
-                protos=protos)
-
-    # Generate event implementation
-    if args.gen_event_cpp:
-        curdir = os.path.dirname(os.path.abspath(__file__))
-        template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.cpp'])
-        output_fullpath = os.sep.join([output_dir, output_filename])
-
-        MakoTemplateWriter.to_file(template_file, output_fullpath,
-                cmdline=sys.argv,
-                filename=output_filename,
-                protos=protos)
-
-    # Generate event handler header
-    if args.gen_eventhandler_hpp:
-        curdir = os.path.dirname(os.path.abspath(__file__))
-        template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandler.hpp'])
-        output_fullpath = os.sep.join([output_dir, output_filename])
-
-        MakoTemplateWriter.to_file(template_file, output_fullpath,
-                cmdline=sys.argv,
-                filename=output_filename,
-                event_header='gen_ar_event.hpp',
-                protos=protos)
-
-    # Generate event handler header
-    if args.gen_eventhandlerfile_hpp:
-        curdir = os.path.dirname(os.path.abspath(__file__))
-        template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandlerfile.hpp'])
-        output_fullpath = os.sep.join([output_dir, output_filename])
-
-        MakoTemplateWriter.to_file(template_file, output_fullpath,
-                cmdline=sys.argv,
-                filename=output_filename,
-                event_header='gen_ar_eventhandler.hpp',
-                protos=protos)
-
-    return 0
+    rval = 0
+
+    try:
+        # Generate event header
+        if args.gen_event_hpp:
+            curdir = os.path.dirname(os.path.abspath(__file__))
+            template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.hpp'])
+            output_fullpath = os.sep.join([output_dir, output_filename])
+
+            MakoTemplateWriter.to_file(template_file, output_fullpath,
+                    cmdline=sys.argv,
+                    filename=output_filename,
+                    protos=protos)
+
+        # Generate event implementation
+        if args.gen_event_cpp:
+            curdir = os.path.dirname(os.path.abspath(__file__))
+            template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.cpp'])
+            output_fullpath = os.sep.join([output_dir, output_filename])
+
+            MakoTemplateWriter.to_file(template_file, output_fullpath,
+                    cmdline=sys.argv,
+                    filename=output_filename,
+                    protos=protos)
+
+        # Generate event handler header
+        if args.gen_eventhandler_hpp:
+            curdir = os.path.dirname(os.path.abspath(__file__))
+            template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandler.hpp'])
+            output_fullpath = os.sep.join([output_dir, output_filename])
+
+            MakoTemplateWriter.to_file(template_file, output_fullpath,
+                    cmdline=sys.argv,
+                    filename=output_filename,
+                    event_header='gen_ar_event.hpp',
+                    protos=protos)
+
+        # Generate event handler header
+        if args.gen_eventhandlerfile_hpp:
+            curdir = os.path.dirname(os.path.abspath(__file__))
+            template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandlerfile.hpp'])
+            output_fullpath = os.sep.join([output_dir, output_filename])
+
+            MakoTemplateWriter.to_file(template_file, output_fullpath,
+                    cmdline=sys.argv,
+                    filename=output_filename,
+                    event_header='gen_ar_eventhandler.hpp',
+                    protos=protos)
+
+        rval = CopyDirFilesIfDifferent(output_dir, final_output_dir)
+
+    except:
+        rval = 1
+
+    finally:
+        DeleteDirTree(output_dir)
+
+    return rval
 
 if __name__ == '__main__':
     sys.exit(main())
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
index 414a04e38a..2931ce816b 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_backends.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
+# Copyright (C) 2017-2018 Intel Corporation.   All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the 'Software'),
@@ -26,7 +26,7 @@ from __future__ import print_function
 import itertools
 import os
 import sys
-from gen_common import ArgumentParser, MakoTemplateWriter
+from gen_common import *
 
 
 def main(args=sys.argv[1:]):
@@ -100,46 +100,67 @@ def main(args=sys.argv[1:]):
     linesPerFile = (len(output_list) + numFiles - 1) // numFiles
     chunkedList = [output_list[x:x+linesPerFile] for x in range(0, len(output_list), linesPerFile)]
 
+    tmp_output_dir = MakeTmpDir('_codegen')
+
+    if not os.path.exists(args.outdir):
+        try:
+            os.makedirs(args.outdir)
+        except OSError as err:
+            if err.errno != errno.EEXIST:
+                print('ERROR: Could not create directory:', args.outdir, file=sys.stderr)
+                return 1
+
+    rval = 0
+
     # generate .cpp files
-    if args.cpp:
-        baseCppName = os.path.join(args.outdir, backend.outFileName)
-        templateCpp = os.path.join(thisDir, 'templates', backend.template)
+    try:
+        if args.cpp:
+            baseCppName = os.path.join(tmp_output_dir, backend.outFileName)
+            templateCpp = os.path.join(thisDir, 'templates', backend.template)
+
+            for fileNum in range(numFiles):
+                filename = baseCppName % str(fileNum)
+                MakoTemplateWriter.to_file(
+                    templateCpp,
+                    baseCppName % str(fileNum),
+                    cmdline=sys.argv,
+                    fileNum=fileNum,
+                    funcList=chunkedList[fileNum])
+
+        if args.hpp:
+            baseHppName = os.path.join(tmp_output_dir, backend.outHeaderName)
+            templateHpp = os.path.join(thisDir, 'templates', backend.hpp_template)
 
-        for fileNum in range(numFiles):
-            filename = baseCppName % str(fileNum)
             MakoTemplateWriter.to_file(
-                templateCpp,
-                baseCppName % str(fileNum),
+                templateHpp,
+                baseHppName,
                 cmdline=sys.argv,
-                fileNum=fileNum,
-                funcList=chunkedList[fileNum])
-
-    if args.hpp:
-        baseHppName = os.path.join(args.outdir, backend.outHeaderName)
-        templateHpp = os.path.join(thisDir, 'templates', backend.hpp_template)
-
-        MakoTemplateWriter.to_file(
-            templateHpp,
-            baseHppName,
-            cmdline=sys.argv,
-            numFiles=numFiles,
-            filename=backend.outHeaderName,
-            tableName=backend.tableName)
-
-    # generate gen_backend.cmake file
-    if args.cmake:
-        templateCmake = os.path.join(thisDir, 'templates', 'gen_backend.cmake')
-        cmakeFile = os.path.join(args.outdir, backend.cmakeFileName)
-
-        MakoTemplateWriter.to_file(
-            templateCmake,
-            cmakeFile,
-            cmdline=sys.argv,
-            srcVar=backend.cmakeSrcVar,
-            numFiles=numFiles,
-            baseCppName='${RASTY_GEN_SRC_DIR}/backends/' + os.path.basename(baseCppName))
-
-    return 0
+                numFiles=numFiles,
+                filename=backend.outHeaderName,
+                tableName=backend.tableName)
+
+        # generate gen_backend.cmake file
+        if args.cmake:
+            templateCmake = os.path.join(thisDir, 'templates', 'gen_backend.cmake')
+            cmakeFile = os.path.join(tmp_output_dir, backend.cmakeFileName)
+
+            MakoTemplateWriter.to_file(
+                templateCmake,
+                cmakeFile,
+                cmdline=sys.argv,
+                srcVar=backend.cmakeSrcVar,
+                numFiles=numFiles,
+                baseCppName='${RASTY_GEN_SRC_DIR}/backends/' + os.path.basename(baseCppName))
+
+        rval = CopyDirFilesIfDifferent(tmp_output_dir, args.outdir)
+
+    except:
+        rval = 1
+
+    finally:
+        DeleteDirTree(tmp_output_dir)
+
+    return rval
 
 if __name__ == '__main__':
     sys.exit(main())
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py
index 7f53ec6ad6..44a0cc88e3 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_common.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
+# Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -25,9 +25,128 @@ import os
 import errno
 import sys
 import argparse
+import tempfile
+import filecmp
+import shutil
 from mako.template import Template
 from mako.exceptions import RichTraceback
 
+#==============================================================================
+def MakeTmpDir(suffix=''):
+    '''
+        Create temporary directory for use in codegen scripts.
+    '''
+    return tempfile.mkdtemp(suffix)
+
+#==============================================================================
+def MakeDir(dir_path):
+    '''
+        Create a directory if it doesn't exist
+
+        returns 0 on success, non-zero on failure
+    '''
+    dir_path = os.path.abspath(dir_path)
+
+    if not os.path.exists(dir_path):
+        try:
+            os.makedirs(dir_path)
+        except OSError as err:
+            if err.errno != errno.EEXIST:
+                return 1
+    else:
+        if not os.path.isdir(dir_path):
+            return 1
+
+    return 0
+
+#==============================================================================
+def DeleteDirTree(dir_path):
+    '''
+        Delete directory tree.
+
+        returns 0 on success, non-zero on failure
+    '''
+    rval = 0
+    try:
+        shutil.rmtree(dir_path, False)
+    except:
+        rval = 1
+    return rval
+
+#==============================================================================
+def CopyFileIfDifferent(src, dst, verbose = False):
+    '''
+        Copy <src> file to <dst> file if the <dst>
+        file either doesn't contain the file or the file
+        contents are different.
+
+        returns 0 on success, non-zero on failure
+    '''
+
+    assert os.path.isfile(src)
+    assert (False == os.path.exists(dst) or os.path.isfile(dst))
+
+    need_copy = not os.path.exists(dst)
+    if not need_copy:
+        need_copy = not filecmp.cmp(src, dst)
+
+    if need_copy:
+        try:
+            shutil.copy2(src, dst)
+        except:
+            print('ERROR: Could not copy %s to %s' % (src, dst), file=sys.stderr)
+            return 1
+
+        if verbose:
+            print(src, '-->', dst)
+
+    return 0
+
+#==============================================================================
+def CopyDirFilesIfDifferent(src, dst, recurse = True, verbose = False, orig_dst = None):
+    '''
+        Copy files <src> directory to <dst> directory if the <dst>
+        directory either doesn't contain the file or the file
+        contents are different.
+
+        Optionally recurses into subdirectories
+
+        returns 0 on success, non-zero on failure
+    '''
+
+    assert os.path.isdir(src)
+    assert os.path.isdir(dst)
+
+    src = os.path.abspath(src)
+    dst = os.path.abspath(dst)
+
+    if not orig_dst:
+        orig_dst = dst
+
+    for f in os.listdir(src):
+        src_path = os.path.join(src, f)
+        dst_path = os.path.join(dst, f)
+
+        # prevent recursion
+        if src_path == orig_dst:
+            continue
+
+        if os.path.isdir(src_path):
+            if recurse:
+                if MakeDir(dst_path):
+                    print('ERROR: Could not create directory:', dst_path, file=sys.stderr)
+                    return 1
+
+                if verbose:
+                    print('mkdir', dst_path)
+                rval = CopyDirFilesIfDifferent(src_path, dst_path, recurse, verbose, orig_dst)
+        else:
+            rval = CopyFileIfDifferent(src_path, dst_path, verbose)
+
+        if rval:
+            return rval
+
+    return 0
 
 #==============================================================================
 class MakoTemplateWriter:
@@ -57,20 +176,18 @@ class MakoTemplateWriter:
                 print('File %s, line %s, in %s' % (filename, lineno, function))
                 print(line, '\n')
             print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
+            raise
 
     @staticmethod
     def to_file(template_filename, output_filename, **kwargs):
         '''
             Write template data to a file
         '''
-        if not os.path.exists(os.path.dirname(output_filename)):
-            try:
-                os.makedirs(os.path.dirname(output_filename))
-            except OSError as err:
-                if err.errno != errno.EEXIST:
-                    raise
+        if MakeDir(os.path.dirname(output_filename)):
+            return 1
         with open(output_filename, 'w') as outfile:
             print(MakoTemplateWriter.to_string(template_filename, **kwargs), file=outfile)
+        return 0
 
 
 #==============================================================================
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
index 33f62a28ce..7733f86ec2 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_knobs.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
+# Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -24,7 +24,7 @@ from __future__ import print_function
 import os
 import sys
 import knob_defs
-from gen_common import MakoTemplateWriter, ArgumentParser
+from gen_common import *
 
 def main(args=sys.argv[1:]):
 
@@ -40,22 +40,39 @@ def main(args=sys.argv[1:]):
     template_cpp = os.path.join(cur_dir, 'templates', 'gen_knobs.cpp')
     template_h = os.path.join(cur_dir, 'templates', 'gen_knobs.h')
 
-    if args.gen_h:
-        MakoTemplateWriter.to_file(
-            template_h,
-            args.output,
-            cmdline=sys.argv,
-            filename='gen_knobs',
-            knobs=knob_defs.KNOBS)
-
-    if args.gen_cpp:
-        MakoTemplateWriter.to_file(
-            template_cpp,
-            args.output,
-            cmdline=sys.argv,
-            filename='gen_knobs',
-            knobs=knob_defs.KNOBS,
-            includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'])
+    output_filename = os.path.basename(args.output)
+    output_dir = MakeTmpDir('_codegen')
+
+    output_file = os.path.join(output_dir, output_filename)
+
+    rval = 0
+
+    try:
+        if args.gen_h:
+            MakoTemplateWriter.to_file(
+                template_h,
+                output_file,
+                cmdline=sys.argv,
+                filename='gen_knobs',
+                knobs=knob_defs.KNOBS)
+
+        if args.gen_cpp:
+            MakoTemplateWriter.to_file(
+                template_cpp,
+                output_file,
+                cmdline=sys.argv,
+                filename='gen_knobs',
+                knobs=knob_defs.KNOBS,
+                includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'])
+
+        rval = CopyFileIfDifferent(output_file, args.output)
+
+    except:
+        rval = 1
+
+    finally:
+        # ignore errors from delete of tmp directory
+        DeleteDirTree(output_dir)
 
     return 0
 
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
index 4a7d2e9a54..9c1e9e0ac8 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 Intel Corporation.   All Rights Reserved.
+# Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -21,7 +21,7 @@
 
 from __future__ import print_function
 import os, sys, re
-from gen_common import MakoTemplateWriter, ArgumentParser
+from gen_common import *
 from argparse import FileType
 
 inst_aliases = {
@@ -312,21 +312,37 @@ def main():
     if not os.path.exists(args.output):
         os.makedirs(args.output)
 
-    if args.input:
-        functions = parse_ir_builder(args.input)
+    final_output_dir = args.output
+    args.output = MakeTmpDir('_codegen')
 
-        if args.gen_h:
-            generate_gen_h(functions, args.output)
+    rval = 0
+    try:
+        if args.input:
+            functions = parse_ir_builder(args.input)
 
-    elif args.gen_h:
-        print('Need to specify --input for --gen_h!')
+            if args.gen_h:
+                generate_gen_h(functions, args.output)
 
-    if args.gen_meta_h:
-        generate_meta_h(args.output)
+        elif args.gen_h:
+            print('Need to specify --input for --gen_h!')
 
-    if args.gen_intrin_h:
-        generate_intrin_h(args.output)
+        if args.gen_meta_h:
+            generate_meta_h(args.output)
+
+        if args.gen_intrin_h:
+            generate_intrin_h(args.output)
+
+        rval = CopyDirFilesIfDifferent(args.output, final_output_dir)
+
+    except:
+        print('ERROR: Could not generate llvm_ir_macros', file=sys.stderr)
+        rval = 1
+
+    finally:
+        DeleteDirTree(args.output)
+
+    return rval
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())
 # END OF FILE
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
index d8863c07e3..8650220c08 100644
--- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
+++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_types.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 Intel Corporation.   All Rights Reserved.
+# Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -21,7 +21,7 @@
 
 from __future__ import print_function
 import os, sys, re
-from gen_common import MakoTemplateWriter, ArgumentParser
+from gen_common import *
 from argparse import FileType
 
 '''
@@ -333,8 +333,29 @@ def main():
             help='Path to output file', required=True)
     args = parser.parse_args()
 
-    gen_llvm_types(args.input, args.output)
+    final_output_dir = os.path.dirname(args.output)
+    if MakeDir(final_output_dir):
+        return 1
+
+    final_output_file = args.output
+
+    tmp_dir = MakeTmpDir('_codegen')
+    args.output = os.path.join(tmp_dir, os.path.basename(args.output))
+
+    rval = 0
+    try:
+        gen_llvm_types(args.input, args.output)
+
+        rval = CopyFileIfDifferent(args.output, final_output_file)
+    except:
+        print('ERROR: Could not generate llvm types', file=sys.stderr)
+        rval = 1
+
+    finally:
+        DeleteDirTree(tmp_dir)
+
+    return rval
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())
 # END OF FILE