Merge remote-tracking branch 'aosp/upstream' into master

390115c Handle nested define/endef
c15a824 Update findleaves.py and add a few testcases
af13468 Merge pull request #73 from colincross/findleaves
f0a6fdf [C++] Add support for multiple filenames to findleaves emulation
1c3a695 Support all kinds of command line variables
ac6f169 Allow NULL filename for -d flag
c58db9a [C++] Add -d flag to make debugging slightly easier
38892d8 Normalize log for recent ninja
9f6343c Always sort glob results
8082dcb Skip 3 tests which fail with make 4
1561f68 Skip shell_var_with_args.mk with GNU make 4
a1ded1c Use override in posix_var.mk to fix test with make 4
fc24bd2 Stop overwriting /bin/sh by bash on travis
913eb74 Normalize Unicode quotes for recent find
2d2ed95 Explicitly use SHELL=/bin/bash
d07e297 [C++] Stop using an uninitialized value
855fea4 Fix multi_implicit_output_patterns.mk for GNU make 4
dc258cb Clean up normalization in runtest.rb a bit
3009771 Merge pull request #66 from stefanb2/topic-issue-65
bfae810 Remove test output in ckati_clean
a24a7a0 Normalize GNU make 4.00 in runtest.rb
e4e56f3 Suppress GNU make jobserver magic in runtest.rb
2941ea0 [C++] Handle .POSIX at eval time
03fa345 Add testcase/posix_var.mk
a8fafa4 Remove Go related targets from test and clean
d8d43ea [C++] Reduce unnecessary string allocation
c6926b0 Add testcase/call_with_whitespace.mk
5dfb361 Add testcase/recursive_marker.mk
02e5ee3 [C++] Remove unnecessary #include
6823600 Add a test case to override_override.mk
4e60e5f Merge pull request #58 from stefanb2/topic-issue-57
952d445 [C++] Ignore recursive marker in recipes
786881c [C++] Replace erroneous return in EvalInclude()
d4f2871 [C++] Store SHELL value in command result
29b9b74 [C++] Honor "override" when setting global variable
167e1f7 [C++] Ignore white space around $(call) function name
187bf08 [C++] Add support for .POSIX:
cd060c5 add Stefan to CONTRIBUTORS

Change-Id: I7697703aa2a0eb37202645b1895899807e6426b2

38 files changed, 361 insertions, 59 deletions

diff --git a/.travis.yml b/.travis.yml
index 9c785dc..f045dd3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,11 +9,9 @@ before_script:
   - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ trusty main universe"
   - sudo apt-get update -qq
   - sudo apt-get install -y libstdc++-4.8-dev clang-3.5 ninja-build realpath
-  - sudo ln -sf bash /bin/sh
 
 script:
   - make -j4 ckati
   - ruby runtest.rb -c
   - ruby runtest.rb -c -n
   - ruby runtest.rb -c -n -a
- 
\ No newline at end of file
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 523d395..1d81fb2 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -28,5 +28,6 @@ Fumitoshi Ukai <ukai@google.com>
 Kouhei Sutou <kou@cozmixng.org>
 Ryo Hashimoto <hashimoto@google.com>
 Shinichiro Hamaji <hamaji@google.com>
+Stefan Becker <stefanb@gpartner-nvidia.com>
 Steve McKay <smckay@google.com>
 Taiju Tsuiki <tzik@google.com>
diff --git a/Makefile b/Makefile
index 5d0d45c..2754488 100644
--- a/Makefile
+++ b/Makefile
@@ -17,9 +17,9 @@ all: ckati ckati_tests
 include Makefile.kati
 include Makefile.ckati
 
-test: all ckati_tests go_test
-	ruby runtest.rb
+test: all ckati_tests
+	ruby runtest.rb -c -n
 
-clean: ckati_clean go_clean
+clean: ckati_clean
 
 .PHONY: test clean ckati_tests
diff --git a/Makefile.ckati b/Makefile.ckati
index 377e36d..df4c6a5 100644
--- a/Makefile.ckati
+++ b/Makefile.ckati
@@ -126,6 +126,7 @@ ckati_clean:
 	rm -rf $(KATI_INTERMEDIATES_PATH)/*.d
 	rm -rf $(KATI_INTERMEDIATES_PATH)/version.cc
 	rm -rf $(KATI_CXX_TEST_EXES)
+	rm -rf out
 
 .PHONY: ckati_clean
 
diff --git a/command.cc b/command.cc
index f75a8a0..8907630 100644
--- a/command.cc
+++ b/command.cc
@@ -160,6 +160,8 @@ void ParseCommandPrefixes(StringPiece* s, bool* echo, bool* ignore_error) {
       *echo = false;
     } else if (c == '-') {
       *ignore_error = true;
+    } else if (c == '+') {
+      // ignore recursion marker
     } else {
       break;
     }
diff --git a/dep.cc b/dep.cc
index da209c7..502ec47 100644
--- a/dep.cc
+++ b/dep.cc
@@ -25,6 +25,7 @@
 
 #include "eval.h"
 #include "fileutil.h"
+#include "flags.h"
 #include "log.h"
 #include "rule.h"
 #include "stats.h"
@@ -214,6 +215,8 @@ struct RuleMerger {
       if (r == primary_rule)
         continue;
       FillDepNodeFromRule(output, r, n);
+      if (n->loc.filename == NULL)
+        n->loc = r->loc;
     }
   }
 };
@@ -287,7 +290,6 @@ class DepBuilder {
       ".EXPORT_ALL_VARIABLES",
       ".NOTPARALLEL",
       ".ONESHELL",
-      ".POSIX",
       NULL
     };
     for (const char** p = kUnsupportedBuiltinTargets; *p; p++) {
diff --git a/eval.cc b/eval.cc
index 6322fc1..5c9055c 100644
--- a/eval.cc
+++ b/eval.cc
@@ -34,7 +34,9 @@ Evaluator::Evaluator()
     : last_rule_(NULL),
       current_scope_(NULL),
       avoid_io_(false),
-      eval_depth_(0) {
+      eval_depth_(0),
+      posix_sym_(Intern(".POSIX")),
+      is_posix_(false) {
 }
 
 Evaluator::~Evaluator() {
@@ -48,6 +50,7 @@ Var* Evaluator::EvalRHS(Symbol lhs, Value* rhs_v, StringPiece orig_rhs,
                         AssignOp op, bool is_override) {
   VarOrigin origin = (
       (is_bootstrap_ ? VarOrigin::DEFAULT :
+       is_commandline_ ? VarOrigin::COMMAND_LINE :
        is_override ? VarOrigin::OVERRIDE : VarOrigin::FILE));
 
   Var* rhs = NULL;
@@ -101,7 +104,8 @@ void Evaluator::EvalAssign(const AssignStmt* stmt) {
   Var* rhs = EvalRHS(lhs, stmt->rhs, stmt->orig_rhs, stmt->op,
                      stmt->directive == AssignDirective::OVERRIDE);
   if (rhs)
-    lhs.SetGlobalVar(rhs);
+    lhs.SetGlobalVar(rhs,
+                     stmt->directive == AssignDirective::OVERRIDE);
 }
 
 void Evaluator::EvalRule(const RuleStmt* stmt) {
@@ -125,6 +129,11 @@ void Evaluator::EvalRule(const RuleStmt* stmt) {
       rule->cmds.push_back(stmt->after_term);
     }
 
+    for (Symbol o : rule->outputs) {
+      if (o == posix_sym_)
+        is_posix_ = true;
+    }
+
     LOG("Rule: %s", rule->DebugString().c_str());
     rules_.push_back(rule);
     last_rule_ = rule;
@@ -252,7 +261,7 @@ void Evaluator::EvalInclude(const IncludeStmt* stmt) {
     for (const string& fname : *files) {
       if (!stmt->should_exist && g_flags.ignore_optional_include_pattern &&
           Pattern(g_flags.ignore_optional_include_pattern).Match(fname)) {
-        return;
+        continue;
       }
       DoInclude(fname);
     }
@@ -310,6 +319,22 @@ string Evaluator::EvalVar(Symbol name) {
   return LookupVar(name)->Eval(this);
 }
 
+string Evaluator::GetShell() {
+  return EvalVar(kShellSym);
+}
+
+string Evaluator::GetShellFlag() {
+  // TODO: Handle $(.SHELLFLAGS)
+  return is_posix_ ? "-ec" : "-c";
+}
+
+string Evaluator::GetShellAndFlag() {
+  string shell = GetShell();
+  shell += ' ';
+  shell += GetShellFlag();
+  return shell;
+}
+
 void Evaluator::Error(const string& msg) {
   ERROR("%s:%d: %s", LOCF(loc_), msg.c_str());
 }
diff --git a/eval.h b/eval.h
index 3eede2b..bf8c98a 100644
--- a/eval.h
+++ b/eval.h
@@ -61,6 +61,7 @@ class Evaluator {
   void Error(const string& msg);
 
   void set_is_bootstrap(bool b) { is_bootstrap_ = b; }
+  void set_is_commandline(bool c) { is_commandline_ = c; }
 
   void set_current_scope(Vars* v) { current_scope_ = v; }
 
@@ -89,6 +90,10 @@ class Evaluator {
     eval_depth_--;
   }
 
+  string GetShell();
+  string GetShellFlag();
+  string GetShellAndFlag();
+
  private:
   Var* EvalRHS(Symbol lhs, Value* rhs, StringPiece orig_rhs, AssignOp op,
                bool is_override = false);
@@ -105,6 +110,7 @@ class Evaluator {
 
   Loc loc_;
   bool is_bootstrap_;
+  bool is_commandline_;
 
   bool avoid_io_;
   // This value tracks the nest level of make expressions. For
@@ -115,6 +121,9 @@ class Evaluator {
   // error).
   vector<string> delayed_output_commands_;
 
+  Symbol posix_sym_;
+  bool is_posix_;
+
   static unordered_set<Symbol> used_undefined_vars_;
 };
 
diff --git a/exec.cc b/exec.cc
index afba609..6065169 100644
--- a/exec.cc
+++ b/exec.cc
@@ -44,8 +44,9 @@ const double kProcessing = -1.0;
 class Executor {
  public:
   explicit Executor(Evaluator* ev)
-      : ce_(ev) {
-    shell_ = ev->EvalVar(kShellSym);
+      : ce_(ev),
+        num_commands_(0) {
+    shell_ = ev->GetShellAndFlag();
   }
 
   double ExecNode(DepNode* n, DepNode* needed_by) {
diff --git a/fileutil.cc b/fileutil.cc
index c116f31..0d3c2d6 100644
--- a/fileutil.cc
+++ b/fileutil.cc
@@ -65,7 +65,7 @@ int RunCommand(const string& shell, const string& cmd,
                string* s) {
   string cmd_escaped = cmd;
   EscapeShell(&cmd_escaped);
-  string cmd_with_shell = shell + " -c \"" + cmd_escaped + "\"";
+  string cmd_with_shell = shell + " \"" + cmd_escaped + "\"";
   const char* argv[] = {
     "/bin/sh", "-c", cmd_with_shell.c_str(), NULL
   };
@@ -157,7 +157,7 @@ class GlobCache {
       vector<string>* files = p.first->second = new vector<string>;
       if (strcspn(pat, "?*[\\") != strlen(pat)) {
         glob_t gl;
-        glob(pat, GLOB_NOSORT, NULL, &gl);
+        glob(pat, 0, NULL, &gl);
         for (size_t i = 0; i < gl.gl_pathc; i++) {
           files->push_back(gl.gl_pathv[i]);
         }
diff --git a/find.cc b/find.cc
index b783acb..71cc77c 100644
--- a/find.cc
+++ b/find.cc
@@ -262,11 +262,12 @@ class DirentDirNode : public DirentNode {
         if (!c->RunFind(fc, d + 1, path, cur_read_dirs, out))
           return false;
         path->resize(orig_path_size);
-        // Found a leaf, stop the search.
-        if (orig_out_size != out->size())
-          return true;
       }
 
+      // Found a leaf, stop the search.
+      if (orig_out_size != out->size())
+        return true;
+
       for (const auto& p : children_) {
         DirentNode* c = p.second;
         if (!c->IsDirectory())
@@ -593,14 +594,30 @@ class FindCommandParser {
     fc_->type = FindCommandType::FINDLEAVES;
     fc_->follows_symlinks = true;
     StringPiece tok;
+    vector<StringPiece> findfiles;
     while (true) {
       if (!GetNextToken(&tok))
         return false;
       if (tok.empty()) {
-        if (fc_->finddirs.size() < 2)
-          return false;
-        fc_->print_cond.reset(new NameCond(fc_->finddirs.back().as_string()));
-        fc_->finddirs.pop_back();
+        if (fc_->finddirs.size() == 0) {
+          // backwards compatibility
+          if (findfiles.size() < 2)
+            return false;
+          fc_->finddirs.swap(findfiles);
+          fc_->print_cond.reset(new NameCond(fc_->finddirs.back().as_string()));
+          fc_->finddirs.pop_back();
+        } else {
+          if (findfiles.size() < 1)
+            return false;
+          for (auto& file : findfiles) {
+            FindCond* cond = new NameCond(file.as_string());
+            if (fc_->print_cond.get()) {
+              cond = new OrCond(fc_->print_cond.release(), cond);
+            }
+            CHECK(!fc_->print_cond.get());
+            fc_->print_cond.reset(cond);
+          }
+        }
         return true;
       }
 
@@ -621,11 +638,14 @@ class FindCommandParser {
           return false;
         }
         fc_->mindepth = d;
+      } else if (HasPrefix(tok, "--dir=")) {
+        StringPiece dir= tok.substr(strlen("--dir="));
+        fc_->finddirs.push_back(dir);
       } else if (HasPrefix(tok, "--")) {
         WARN("Unknown flag in findleaves.py: %.*s", SPF(tok));
         return false;
       } else {
-        fc_->finddirs.push_back(tok);
+        findfiles.push_back(tok);
       }
     }
   }
diff --git a/flags.cc b/flags.cc
index 7a995a5..876e855 100644
--- a/flags.cc
+++ b/flags.cc
@@ -65,6 +65,8 @@ void Flags::Parse(int argc, char** argv) {
       is_dry_run = true;
     } else if (!strcmp(arg, "-s")) {
       is_silent_mode = true;
+    } else if (!strcmp(arg, "-d")) {
+      enable_debug = true;
     } else if (!strcmp(arg, "--kati_stats")) {
       enable_stat_logs = true;
     } else if (!strcmp(arg, "--warn")) {
diff --git a/flags.h b/flags.h
index 910acbf..a0c6a3b 100644
--- a/flags.h
+++ b/flags.h
@@ -27,6 +27,7 @@ struct Flags {
   bool detect_android_echo;
   bool detect_depfiles;
   bool dump_kati_stamp;
+  bool enable_debug;
   bool enable_kati_warnings;
   bool enable_stat_logs;
   bool gen_all_targets;
diff --git a/func.cc b/func.cc
index 876b274..a7d1c8c 100644
--- a/func.cc
+++ b/func.cc
@@ -318,7 +318,6 @@ void WildcardFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
   for (StringPiece tok : WordScanner(pat)) {
     ScopedTerminator st(tok);
     Glob(tok.data(), &files);
-    sort(files->begin(), files->end());
     for (const string& file : *files) {
       ww.Write(file);
     }
@@ -562,13 +561,14 @@ void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
     return;
   }
 
-  const string&& shell = ev->EvalVar(kShellSym);
+  const string&& shell = ev->GetShellAndFlag();
 
   string out;
   FindCommand* fc = NULL;
   ShellFuncImpl(shell, cmd, &out, &fc);
   if (ShouldStoreCommandResult(cmd)) {
     CommandResult* cr = new CommandResult();
+    cr->shell = shell;
     cr->cmd = cmd;
     cr->find.reset(fc);
     cr->result = out;
@@ -583,11 +583,12 @@ void CallFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
     Intern("5"), Intern("6"),  Intern("7"), Intern("8"), Intern("9")
   };
 
-  const string&& func_name = args[0]->Eval(ev);
+  const string&& func_name_buf = args[0]->Eval(ev);
+  const StringPiece func_name = TrimSpace(func_name_buf);
   Var* func = ev->LookupVar(Intern(func_name));
   if (!func->IsDefined()) {
     KATI_WARN("%s:%d: *warning*: undefined user function: %s",
-              ev->loc(), func_name.c_str());
+              ev->loc(), func_name.as_string().c_str());
   }
   vector<unique_ptr<SimpleVar>> av;
   for (size_t i = 1; i < args.size(); i++) {
diff --git a/func.h b/func.h
index 8db2c7a..e78deb7 100644
--- a/func.h
+++ b/func.h
@@ -42,6 +42,7 @@ FuncInfo* GetFuncInfo(StringPiece name);
 struct FindCommand;
 
 struct CommandResult {
+  string shell;
   string cmd;
   unique_ptr<FindCommand> find;
   string result;
diff --git a/main.cc b/main.cc
index 2627bfd..5afe2b1 100644
--- a/main.cc
+++ b/main.cc
@@ -154,9 +154,14 @@ static int Run(const vector<Symbol>& targets,
   }
   ev->set_is_bootstrap(false);
 
+  ev->set_is_commandline(true);
   for (StringPiece l : cl_vars) {
-    SetVar(l, VarOrigin::COMMAND_LINE);
+    vector<Stmt*> asts;
+    Parse(Intern(l).str(), Loc("*bootstrap*", 0), &asts);
+    CHECK(asts.size() == 1);
+    asts[0]->Eval(ev);
   }
+  ev->set_is_commandline(false);
 
   {
     ScopedTimeReporter tr("eval time");
diff --git a/ninja.cc b/ninja.cc
index d2dc702..acf5293 100644
--- a/ninja.cc
+++ b/ninja.cc
@@ -184,7 +184,8 @@ class NinjaGenerator {
         start_time_(start_time),
         default_target_(NULL) {
     ev_->set_avoid_io(true);
-    shell_ = EscapeNinja(ev->EvalVar(kShellSym));
+    shell_ = EscapeNinja(ev->GetShell());
+    shell_flags_ = EscapeNinja(ev->GetShellFlag());
     const string use_goma_str = ev->EvalVar(Intern("USE_GOMA"));
     use_goma_ = !(use_goma_str.empty() || use_goma_str == "false");
     if (g_flags.goma_dir)
@@ -482,6 +483,10 @@ class NinjaGenerator {
 
     string rule_name = "phony";
     bool use_local_pool = false;
+    if (g_flags.enable_debug) {
+      *o << "# " << (node->loc.filename ? node->loc.filename : "(null)")
+         << ':' << node->loc.lineno << "\n";
+    }
     if (!commands.empty()) {
       rule_name = StringPrintf("rule%d", nn->rule_id);
       *o << "rule " << rule_name << "\n";
@@ -501,7 +506,8 @@ class NinjaGenerator {
         *o << " command = " << shell_ << " $out.rsp\n";
       } else {
         EscapeShell(&cmd_buf);
-        *o << " command = " << shell_ << " -c \"" << cmd_buf << "\"\n";
+        *o << " command = " << shell_ << ' ' << shell_flags_
+           << " \"" << cmd_buf << "\"\n";
       }
       if (node->is_restat) {
         *o << " restat = 1\n";
@@ -727,6 +733,7 @@ class NinjaGenerator {
     const vector<CommandResult*>& crs = GetShellCommandResults();
     DumpInt(fp, crs.size());
     for (CommandResult* cr : crs) {
+      DumpString(fp, cr->shell);
       DumpString(fp, cr->cmd);
       DumpString(fp, cr->result);
       if (!cr->find.get()) {
@@ -769,6 +776,7 @@ class NinjaGenerator {
   bool use_goma_;
   string gomacc_;
   string shell_;
+  string shell_flags_;
   map<string, string> used_envs_;
   string kati_binary_;
   const double start_time_;
diff --git a/parser.cc b/parser.cc
index e6c652a..61d1ac5 100644
--- a/parser.cc
+++ b/parser.cc
@@ -51,6 +51,7 @@ class Parser {
         state_(ParserState::NOT_AFTER_RULE),
         stmts_(stmts),
         out_stmts_(stmts),
+        num_define_nest_(0),
         num_if_nest_(0),
         loc_(filename, 0),
         fixed_lineno_(false) {
@@ -281,6 +282,7 @@ class Parser {
       return;
     }
     define_name_ = line;
+    num_define_nest_ = 1;
     define_start_ = 0;
     define_start_line_ = loc_.lineno;
     state_ = ParserState::NOT_AFTER_RULE;
@@ -288,7 +290,12 @@ class Parser {
 
   void ParseInsideDefine(StringPiece line) {
     line = TrimLeftSpace(line);
-    if (GetDirective(line) != "endef") {
+    StringPiece directive = GetDirective(line);
+    if (directive == "define")
+      num_define_nest_++;
+    else if (directive == "endef")
+      num_define_nest_--;
+    if (num_define_nest_ > 0) {
       if (define_start_ == 0)
         define_start_ = l_;
       return;
@@ -516,6 +523,7 @@ class Parser {
   vector<Stmt*>* out_stmts_;
 
   StringPiece define_name_;
+  int num_define_nest_;
   size_t define_start_;
   int define_start_line_;
 
diff --git a/regen.cc b/regen.cc
index 23151b4..c72bfb6 100644
--- a/regen.cc
+++ b/regen.cc
@@ -52,6 +52,7 @@ class StampChecker {
   };
 
   struct ShellResult {
+    string shell;
     string cmd;
     string result;
     vector<string> missing_dirs;
@@ -230,6 +231,7 @@ class StampChecker {
     for (int i = 0; i < num_crs; i++) {
       ShellResult* sr = new ShellResult;
       commands_.push_back(sr);
+      LOAD_STRING(fp, &sr->shell);
       LOAD_STRING(fp, &sr->cmd);
       LOAD_STRING(fp, &sr->result);
       sr->has_condition = LOAD_INT(fp);
@@ -261,7 +263,6 @@ class StampChecker {
     COLLECT_STATS("glob time (regen)");
     vector<string>* files;
     Glob(gr->pat.c_str(), &files);
-    sort(files->begin(), files->end());
     bool needs_regen = files->size() != gr->result.size();
     for (size_t i = 0; i < gr->result.size(); i++) {
       if (!needs_regen) {
@@ -339,7 +340,7 @@ class StampChecker {
 
     COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
     string result;
-    RunCommand("/bin/sh", sr->cmd, RedirectStderr::DEV_NULL, &result);
+    RunCommand(sr->shell, sr->cmd, RedirectStderr::DEV_NULL, &result);
     FormatForCommandSubstitution(&result);
     if (sr->result != result) {
       if (g_flags.dump_kati_stamp) {
diff --git a/runtest.rb b/runtest.rb
index ab1e26b..fc594d5 100755
--- a/runtest.rb
+++ b/runtest.rb
@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# coding: binary
 #
 # Copyright 2015 Google Inc. All rights reserved
 #
@@ -16,6 +17,10 @@
 
 require 'fileutils'
 
+# suppress GNU make jobserver magic when calling "make"
+ENV.delete('MAKEFLAGS')
+ENV.delete('MAKELEVEL')
+
 while true
   if ARGV[0] == '-s'
     test_serialization = true
@@ -104,29 +109,38 @@ def normalize_ninja_log(log, mk)
             '*** No rule to make target \\1.')
   log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/,
             'ninja: warning: multiple rules generate \\1.')
+
   if mk =~ /err_error_in_recipe.mk/
     # This test expects ninja fails. Strip ninja specific error logs.
-    log.gsub!(/^FAILED: .*\n/, '')
-    log.gsub!(/^ninja: .*\n/, '')
+    ninja_failed_subst = ''
   elsif mk =~ /\/fail_/
     # Recipes in these tests fail.
-    log.gsub!(/^FAILED: .*/, '*** [test] Error 1')
+    ninja_failed_subst = "*** [test] Error 1\n"
+  end
+  if ninja_failed_subst
+    log.gsub!(/^FAILED: (.*\n\/bin\/bash)?.*\n/, ninja_failed_subst)
     log.gsub!(/^ninja: .*\n/, '')
   end
   log
 end
 
+def normalize_quotes(log)
+  log.gsub!(/[`'"]/, '"')
+  # For recent GNU find, which uses Unicode characters.
+  log.gsub!(/(\xe2\x80\x98|\xe2\x80\x99)/, '"')
+  log
+end
+
 def normalize_make_log(expected, mk, via_ninja)
+  expected = normalize_quotes(expected)
   expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '')
   expected.gsub!(/^make(?:\[\d+\])?: /, '')
   expected = move_circular_dep(expected)
 
   # Normalizations for old/new GNU make.
-  expected.gsub!(/[`'"]/, '"')
-  expected.gsub!(/ (?:commands|recipe) for target /,
-                 ' commands for target ')
-  expected.gsub!(/ (?:commands|recipe) commences /,
-                 ' commands commence ')
+  expected.gsub!(' recipe for target ', ' commands for target ')
+  expected.gsub!(' recipe commences ', ' commands commence ')
+  expected.gsub!('missing rule before recipe.', 'missing rule before commands.')
   expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '')
   expected.gsub!('Extraneous text after', 'extraneous text after')
   # Not sure if this is useful.
@@ -134,7 +148,7 @@ def normalize_make_log(expected, mk, via_ninja)
   # GNU make 4.0 has this output.
   expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '')
   # We treat some warnings as errors.
-  expected.gsub!(/^\/bin\/sh: line 0: /, '')
+  expected.gsub!(/^\/bin\/(ba)?sh: line 0: /, '')
   # We print out some ninja warnings in some tests to match what we expect
   # ninja to produce. Remove them if we're not testing ninja.
   if !via_ninja
@@ -147,11 +161,12 @@ def normalize_make_log(expected, mk, via_ninja)
 end
 
 def normalize_kati_log(output)
+  output = normalize_quotes(output)
   output = move_circular_dep(output)
+
   # kati specific log messages.
   output.gsub!(/^\*kati\*.*\n/, '')
   output.gsub!(/^c?kati: /, '')
-  output.gsub!(/[`'"]/, '"')
   output.gsub!(/\/bin\/sh: ([^:]*): command not found/,
                "\\1: Command not found")
   output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '')
@@ -165,6 +180,8 @@ def normalize_kati_log(output)
   output
 end
 
+bash_var = ' SHELL=/bin/bash'
+
 run_make_test = proc do |mk|
   c = File.read(mk)
   expected_failure = false
@@ -183,6 +200,9 @@ run_make_test = proc do |mk|
       if todos.include?('c-ninja') && ckati && via_ninja
         expected_failure = true
       end
+      if todos.include?('c-exec') && ckati && !via_ninja
+        expected_failure = true
+      end
       if todos.include?('ninja') && via_ninja
         expected_failure = true
       end
@@ -213,8 +233,9 @@ run_make_test = proc do |mk|
       if via_ninja || is_silent_test
         cmd += ' -s'
       end
+      cmd += bash_var
       cmd += " #{tc} 2>&1"
-      res = `#{cmd}`
+      res = IO.popen(cmd, 'r:binary', &:read)
       res = normalize_make_log(res, mk, via_ninja)
       expected += "=== #{tc} ===\n" + res
       expected_files = get_output_filenames
@@ -240,6 +261,7 @@ run_make_test = proc do |mk|
       if is_silent_test
         cmd += ' -s'
       end
+      cmd += bash_var
       if !gen_all_targets || mk =~ /makecmdgoals/
         cmd += " #{tc}"
       end
@@ -322,6 +344,7 @@ run_shell_test = proc do |sh|
     if is_ninja_test
       cmd += ' -s'
     end
+    cmd += bash_var
     expected = IO.popen(cmd, 'r:binary', &:read)
     cleanup
 
@@ -338,6 +361,7 @@ run_shell_test = proc do |sh|
         cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=."
       end
     end
+    cmd += bash_var
 
     output = IO.popen(cmd, 'r:binary', &:read)
 
diff --git a/symtab.cc b/symtab.cc
index fb81bfe..b47522b 100644
--- a/symtab.cc
+++ b/symtab.cc
@@ -59,13 +59,14 @@ Var* Symbol::GetGlobalVar() const {
   return v;
 }
 
-void Symbol::SetGlobalVar(Var* v) const {
+void Symbol::SetGlobalVar(Var* v, bool is_override) const {
   if (static_cast<size_t>(v_) >= g_symbol_data.size()) {
     g_symbol_data.resize(v_ + 1);
   }
   Var* orig = g_symbol_data[v_].gv;
-  if (orig->Origin() == VarOrigin::OVERRIDE ||
-      orig->Origin() == VarOrigin::ENVIRONMENT_OVERRIDE) {
+  if (!is_override &&
+      (orig->Origin() == VarOrigin::OVERRIDE ||
+       orig->Origin() == VarOrigin::ENVIRONMENT_OVERRIDE)) {
     return;
   }
   if (orig->Origin() == VarOrigin::AUTOMATIC) {
diff --git a/symtab.h b/symtab.h
index d1de4e1..e7e71d5 100644
--- a/symtab.h
+++ b/symtab.h
@@ -56,7 +56,7 @@ class Symbol {
   bool IsValid() const { return v_ >= 0; }
 
   Var* GetGlobalVar() const;
-  void SetGlobalVar(Var* v) const;
+  void SetGlobalVar(Var* v, bool is_override = false) const;
 
  private:
   explicit Symbol(int v);
diff --git a/testcase/call_with_whitespace.mk b/testcase/call_with_whitespace.mk
new file mode 100644
index 0000000..f833b53
--- /dev/null
+++ b/testcase/call_with_whitespace.mk
@@ -0,0 +1,9 @@
+func = $(info called with '$(1)')
+test = $(call $(1),$(1))
+
+$(call test,func)
+$(call test, func)
+$(call test,func )
+$(call test, func )
+
+test:
diff --git a/testcase/err_export_override.mk b/testcase/err_export_override.mk
index 54de5b9..6bb9f75 100644
--- a/testcase/err_export_override.mk
+++ b/testcase/err_export_override.mk
@@ -1,5 +1,12 @@
 # TODO(c): Fix - "override export define A" is invalid "override" directive.
 
+# GNU make 4 accepts this syntax. Note kati doesn't agree with make 4
+# either.
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+ifeq ($(MAKE)$(MAKEVER),make4)
+$(error test skipped)
+endif
+
 export override define A
 PASS_A
 endef
diff --git a/testcase/err_override_export.mk b/testcase/err_override_export.mk
index d34a06f..2d72aab 100644
--- a/testcase/err_override_export.mk
+++ b/testcase/err_override_export.mk
@@ -1,5 +1,12 @@
 # TODO(c): Fix - "override export define A" is invalid "override" directive.
 
+# GNU make 4 accepts this syntax. Note kati doesn't agree with make 4
+# either.
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+ifeq ($(MAKE)$(MAKEVER),make4)
+$(error test skipped)
+endif
+
 override export define A
 PASS_A
 endef
diff --git a/testcase/err_unmatched_endef.mk b/testcase/err_unmatched_endef.mk
new file mode 100644
index 0000000..b1a44ce
--- /dev/null
+++ b/testcase/err_unmatched_endef.mk
@@ -0,0 +1,9 @@
+define test1
+# Typo below, endif instead of endef
+endif
+define test2
+endef
+
+foo:
+	echo FAIL
+
diff --git a/testcase/find_command.mk b/testcase/find_command.mk
index a0bbc82..c3e8a07 100644
--- a/testcase/find_command.mk
+++ b/testcase/find_command.mk
@@ -116,6 +116,8 @@ endif
 	$(call run_find, build/tools/findleaves.py --mindepth=2 testdir file1)
 	$(call run_find, build/tools/findleaves.py --mindepth=3 testdir file1)
 	$(call run_find, build/tools/findleaves.py --mindepth=2 testdir file1)
+	$(call run_find, build/tools/findleaves.py --prune=dir1 --dir=testdir file1)
+	$(call run_find, build/tools/findleaves.py --prune=dir1 --dir=testdir file3 link3)
 	@echo missing chdir / testdir
 	$(call run_find, cd xxx && find .)
 	$(call run_find, if [ -d xxx ]; then find .; fi)
diff --git a/testcase/include_glob_order.mk b/testcase/include_glob_order.mk
new file mode 100644
index 0000000..8b337f9
--- /dev/null
+++ b/testcase/include_glob_order.mk
@@ -0,0 +1,19 @@
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+
+# GNU make 4 doesn't sort glob results.
+ifeq ($(MAKEVER,4))
+
+$(info test skipped)
+
+else
+
+test1:
+	echo '$$(info foo)' > foo.d
+	echo '$$(info bar)' > bar.d
+
+test2:
+	echo $(wildcard *.d)
+
+-include *.d
+
+endif
diff --git a/testcase/multi_implicit_output_patterns.mk b/testcase/multi_implicit_output_patterns.mk
index 4032ef4..8b53b49 100644
--- a/testcase/multi_implicit_output_patterns.mk
+++ b/testcase/multi_implicit_output_patterns.mk
@@ -1,5 +1,7 @@
 # TODO(go): Fix
 
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+
 all: a.h.x a.c.x a.h.z a.c.z b.h.x b.c.x b.h.z b.c.z
 
 a.h.%:
@@ -12,7 +14,10 @@ b.h.% b.c.%:
 
 b.h.z: pass
 
+# GNU make 4 invokes this rule.
+ifeq ($(MAKEVER,3))
 b.c.z: fail
+endif
 
 pass:
 	echo PASS
diff --git a/testcase/nested_define.mk b/testcase/nested_define.mk
new file mode 100644
index 0000000..8de444a
--- /dev/null
+++ b/testcase/nested_define.mk
@@ -0,0 +1,21 @@
+define outer
+  define inner
+PASS
+   endef
+  define inner_fail
+FAIL
+    endef
+endef
+
+# Prefixed defines don't increase the nest level.
+define outer_override
+override define inner2
+export define inner3
+endef
+
+A := $(inner_fail)
+$(eval $(outer))
+
+foo:
+	echo $(A)
+	echo $(inner)
diff --git a/testcase/ninja_regen_glob.sh b/testcase/ninja_regen_glob.sh
new file mode 100755
index 0000000..71dca29
--- /dev/null
+++ b/testcase/ninja_regen_glob.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Copyright 2016 Google Inc. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+log=/tmp/log
+mk="$@"
+
+sleep_if_necessary() {
+  if [ x$(uname) != x"Linux" -o x"${TRAVIS}" != x"" ]; then
+    sleep "$@"
+  fi
+}
+
+touch xe.mk yc.mk xa.mk yb.mk xd.mk
+
+cat <<EOF > Makefile
+include *.mk
+all:
+	echo foo
+EOF
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  ./ninja.sh
+fi
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if grep regenerating ${log}; then
+    echo 'Should not be regenerated'
+  fi
+  ./ninja.sh
+fi
diff --git a/testcase/override_override.mk b/testcase/override_override.mk
index 9d23355..81bcdf8 100644
--- a/testcase/override_override.mk
+++ b/testcase/override_override.mk
@@ -6,8 +6,14 @@ PASS_B
 endef
 B:=FAIL_B
 
+override C := FAIL_C
+override C := PASS_C
+C := FAIL_C2
+
 test:
 	echo $(A)
 	echo $(origin A)
 	echo $(B)
 	echo $(origin B)
+	echo $(C)
+	echo $(origin C)
diff --git a/testcase/posix_var.mk b/testcase/posix_var.mk
new file mode 100644
index 0000000..5574f09
--- /dev/null
+++ b/testcase/posix_var.mk
@@ -0,0 +1,21 @@
+# TODO(go): Fix
+
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+
+# GNU make 3.82 has this feature though.
+ifeq ($(MAKEVER),3)
+
+test:
+	echo test skipped
+
+else
+
+$(info $(shell echo foo))
+override SHELL := echo
+$(info $(shell echo bar))
+.POSIX:
+$(info $(shell echo baz))
+test:
+	foobar
+
+endif
diff --git a/testcase/recursive_marker.mk b/testcase/recursive_marker.mk
new file mode 100644
index 0000000..283715f
--- /dev/null
+++ b/testcase/recursive_marker.mk
@@ -0,0 +1,3 @@
+test:
+	+echo PASS
+
diff --git a/testcase/shell_var.mk b/testcase/shell_var.mk
index 8f5de31..b42b0c9 100644
--- a/testcase/shell_var.mk
+++ b/testcase/shell_var.mk
@@ -1,11 +1,11 @@
 $(info $(SHELL))
 
-SHELL:=/bin/echo
+override SHELL:=/bin/echo
 
 $(info $(shell foo))
 
 echo=/bin/echo
-SHELL=$(echo)
+override SHELL=$(echo)
 
 $(info $(shell bar))
 
diff --git a/testcase/shell_var_with_args.mk b/testcase/shell_var_with_args.mk
index ca7b54a..4256eb0 100644
--- a/testcase/shell_var_with_args.mk
+++ b/testcase/shell_var_with_args.mk
@@ -1,9 +1,21 @@
 # TODO(go): Fix
 
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+
+ifeq ($(MAKEVER),4)
+
+# GNU make 4 escapes $(SHELL).
+test:
+	echo test skipped
+
+else
+
 export FOO=-x
 
-SHELL := PS4="cmd: " /bin/bash $${FOO}
+override SHELL := PS4="cmd: " /bin/bash $${FOO}
 $(info $(shell echo foo))
 
 test:
 	@echo baz
+
+endif
diff --git a/testcase/tools/findleaves.py b/testcase/tools/findleaves.py
index 3a9e508..72cc024 100755
--- a/testcase/tools/findleaves.py
+++ b/testcase/tools/findleaves.py
@@ -23,7 +23,7 @@
 import os
 import sys
 
-def perform_find(mindepth, prune, dirlist, filename):
+def perform_find(mindepth, prune, dirlist, filenames):
   result = []
   pruneleaves = set(map(lambda x: os.path.split(x)[1], prune))
   for rootdir in dirlist:
@@ -48,19 +48,24 @@ def perform_find(mindepth, prune, dirlist, filename):
         if depth < mindepth:
           continue
       # match
-      if filename in files:
-        result.append(os.path.join(root, filename))
-        del dirs[:]
+      for filename in filenames:
+        if filename in files:
+          result.append(os.path.join(root, filename))
+          del dirs[:]
   return result
 
 def usage():
-  sys.stderr.write("""Usage: %(progName)s [<options>] <dirlist> <filename>
+  sys.stderr.write("""Usage: %(progName)s [<options>] [--dir=<dir>] <filenames>
 Options:
    --mindepth=<mindepth>
        Both behave in the same way as their find(1) equivalents.
    --prune=<dirname>
        Avoids returning results from inside any directory called <dirname>
        (e.g., "*/out/*"). May be used multiple times.
+   --dir=<dir>
+       Add a directory to search.  May be repeated multiple times.  For backwards
+       compatibility, if no --dir argument is provided then all but the last entry
+       in <filenames> are treated as directories.
 """ % {
       "progName": os.path.split(sys.argv[0])[1],
     })
@@ -69,6 +74,7 @@ Options:
 def main(argv):
   mindepth = -1
   prune = []
+  dirlist = []
   i=1
   while i<len(argv) and len(argv[i])>2 and argv[i][0:2] == "--":
     arg = argv[i]
@@ -82,14 +88,24 @@ def main(argv):
       if len(p) == 0:
         usage()
       prune.append(p)
+    elif arg.startswith("--dir="):
+      d = arg[len("--dir="):]
+      if len(p) == 0:
+        usage()
+      dirlist.append(d)
     else:
       usage()
     i += 1
-  if len(argv)-i < 2: # need both <dirlist> and <filename>
-    usage()
-  dirlist = argv[i:-1]
-  filename = argv[-1]
-  results = list(set(perform_find(mindepth, prune, dirlist, filename)))
+  if len(dirlist) == 0: # backwards compatibility
+    if len(argv)-i < 2: # need both <dirlist> and <filename>
+      usage()
+    dirlist = argv[i:-1]
+    filenames = [argv[-1]]
+  else:
+    if len(argv)-i < 1: # need <filename>
+      usage()
+    filenames = argv[i:]
+  results = list(set(perform_find(mindepth, prune, dirlist, filenames)))
   results.sort()
   for r in results:
     print r
diff --git a/testcase/wildcard_cache.mk b/testcase/wildcard_cache.mk
index 1c9174a..83ba3db 100644
--- a/testcase/wildcard_cache.mk
+++ b/testcase/wildcard_cache.mk
@@ -1,4 +1,11 @@
 # TODO(c): Fix this. Maybe $(wildcard) always runs at eval-phase.
+
+# GNU make 4 agrees with ckati.
+MAKEVER:=$(shell make --version | ruby -n0e 'puts $$_[/Make (\d)/,1]')
+ifeq ($(MAKE)$(MAKEVER),make4)
+$(error test skipped)
+endif
+
 files = $(wildcard *,*)
 
 # if make starts without foo,bar, it will be empty, although expect foo,bar.