Web lists-archives.com

Re: [Samba] [Samba 4.5] Very slow LDAP Queries (almost unusable), performance tunning ?




On Thu, 2017-03-23 at 11:12 +0100, Gaetan SLONGO via samba wrote:
> Dear users, 
> 
> We are facing to a big latency issue regarding the LDAP Server (both
> encrypted & plain). 
> 
> We have a Zarafa mail server which makes a lot of queries and puts a
> samba process to 100% usage. This latency makes the mail server
> unusable.. The mail server was previously on OpenLDAP and there was
> not performance issues. 
> 
> A simple LDAP query can take up to 25 sec to perform !! 

Can you or others who use this please give the attached patch (on git
master) a try?

We fix a major locking issue that makes our searches slow, and provide
multi-process support.

Thanks,

Andrew Bartlett

-- 
Andrew Bartlett
https://samba.org/~abartlet/
Authentication Developer, Samba Team         https://samba.org
Samba Development and Support, Catalyst IT   
https://catalyst.net.nz/services/samba



From 28d1bec88a0e785802726e8b01b7fd9322d5eef2 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Wed, 29 Mar 2017 15:43:44 +1300
Subject: [PATCH 01/19] perf-tests: Add tests running a bind in parallel

We hope to show that with multiple clients, that multiple servers make sense

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 selftest/perf_tests.py                          |  15 +++
 source4/dsdb/tests/python/ad_dc_multi_bind.py   |  48 +++++--
 source4/dsdb/tests/python/ad_dc_multi_ops.py    | 115 +++++++++++++++++
 source4/dsdb/tests/python/ad_dc_multi_search.py | 164 ++++++++++++++++++++++++
 4 files changed, 330 insertions(+), 12 deletions(-)
 create mode 100644 source4/dsdb/tests/python/ad_dc_multi_ops.py
 create mode 100644 source4/dsdb/tests/python/ad_dc_multi_search.py

diff --git a/selftest/perf_tests.py b/selftest/perf_tests.py
index adc4ef95c7e..0c6cd1c473a 100644
--- a/selftest/perf_tests.py
+++ b/selftest/perf_tests.py
@@ -64,3 +64,18 @@ plantestsuite_loadlist("samba4.ldb.multi_connect.python(ad_dc_ntvfs)",
                                              "dsdb/tests/python/ad_dc_multi_bind.py"),
                         'tdb://$PREFIX_ABS/ad_dc_ntvfs/private/sam.ldb'
                         '$LOADLIST', '$LISTOPT'])
+
+plantestsuite_loadlist("samba4.ldap.ad_dc_multi_search.krb5.python(ad_dc_ntvfs)",
+                       "ad_dc_ntvfs",
+                       [python, os.path.join(samba4srcdir,
+                                             "dsdb/tests/python/ad_dc_multi_search.py"),
+                        '$SERVER', '-U"$USERNAME%$PASSWORD"', '-k yes',
+                        '--realm=$REALM',
+                        '$LOADLIST', '$LISTOPT'])
+
+plantestsuite_loadlist("samba4.ldb.multi_search.python(ad_dc_ntvfs)",
+                       "ad_dc_ntvfs",
+                       [python, os.path.join(samba4srcdir,
+                                             "dsdb/tests/python/ad_dc_multi_search.py"),
+                        'tdb://$PREFIX_ABS/ad_dc_ntvfs/private/sam.ldb'
+                        '$LOADLIST', '$LISTOPT'])
diff --git a/source4/dsdb/tests/python/ad_dc_multi_bind.py b/source4/dsdb/tests/python/ad_dc_multi_bind.py
index 10daed548ff..04762dba9fa 100644
--- a/source4/dsdb/tests/python/ad_dc_multi_bind.py
+++ b/source4/dsdb/tests/python/ad_dc_multi_bind.py
@@ -32,10 +32,7 @@ except ImportError:
 
 from samba.samdb import SamDB
 from samba.auth import system_session
-from ldb import Message, MessageElement, Dn, LdbError
-from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
-from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
-
+from ldb import SCOPE_BASE, SCOPE_SUBTREE
 parser = optparse.OptionParser("ad_dc_mulit_bind.py [options] <host>")
 sambaopts = options.SambaOptions(parser)
 parser.add_option_group(sambaopts)
@@ -50,7 +47,6 @@ credopts = options.CredentialsOptions(parser)
 parser.add_option_group(credopts)
 opts, args = parser.parse_args()
 
-
 if len(args) < 1:
     parser.print_usage()
     sys.exit(1)
@@ -60,23 +56,51 @@ host = args[0]
 lp = sambaopts.get_loadparm()
 creds = credopts.get_credentials(lp)
 
-class UserTests(samba.tests.TestCase):
+import ad_dc_multi_ops
 
+class MultiBindTests(ad_dc_multi_ops.MultiOpsTests):
     def setUp(self):
-        super(UserTests, self).setUp()
+        super(MultiBindTests, self).setUp()
         self.lp = lp
 
-    def tearDown(self):
-        super(UserTests, self).tearDown()
+    def test_500_concurrent_binds_1_worker(self):
+        def bind(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_BASE, attrs=["*"])
 
-    def test_1000_binds(self):
+        self._test_concurrent_things(bind, 1)
 
-        for x in range(1, 1000):
+
+    def test_500_concurrent_binds_2_workers(self):
+        def bind(self):
             samdb = SamDB(host, credentials=creds,
-                         session_info=system_session(self.lp), lp=self.lp)
+                          session_info=system_session(self.lp), lp=self.lp)
             samdb.search(base=samdb.domain_dn(),
                          scope=SCOPE_BASE, attrs=["*"])
 
+        self._test_concurrent_things(bind, 2)
+
+
+    def test_500_concurrent_binds_5_workers(self):
+        def bind(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_BASE, attrs=["*"])
+
+        self._test_concurrent_things(bind, 5)
+
+
+    def test_500_concurrent_binds_10_workers(self):
+        def subtree_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(subtree_search, 10)
 
 if "://" not in host:
     if os.path.isfile(host):
diff --git a/source4/dsdb/tests/python/ad_dc_multi_ops.py b/source4/dsdb/tests/python/ad_dc_multi_ops.py
new file mode 100644
index 00000000000..0bbc1d00f4b
--- /dev/null
+++ b/source4/dsdb/tests/python/ad_dc_multi_ops.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import optparse
+import sys
+sys.path.insert(0, 'bin/python')
+
+import os
+import samba
+import samba.getopt as options
+import random
+import tempfile
+import shutil
+import time
+
+# We try to use the test infrastructure of Samba 4.3+, but if it
+# doesn't work, we are probably in a back-ported patch and trying to
+# run on 4.1 or something.
+#
+# Don't copy this horror into ordinary tests -- it is special for
+# performance tests that want to apply to old versions.
+try:
+    from samba.tests.subunitrun import SubunitOptions, TestProgram
+    ANCIENT_SAMBA = False
+except ImportError:
+    ANCIENT_SAMBA = True
+    samba.ensure_external_module("testtools", "testtools")
+    samba.ensure_external_module("subunit", "subunit/python")
+    from subunit.run import SubunitTestRunner
+    import unittest
+
+from samba.messaging import Messaging
+from samba.dcerpc import messaging
+import os, signal
+import time
+
+class MultiOpsTests(samba.tests.TestCase):
+
+    def get_context(self, *args, **kwargs):
+        kwargs['lp_ctx'] = samba.tests.env_loadparm()
+        return Messaging(*args, **kwargs)
+
+    def setUp(self):
+        super(MultiOpsTests, self).setUp()
+
+    def tearDown(self):
+        super(MultiOpsTests, self).tearDown()
+
+    def _test_concurrent_things(self, task, workers, num_ops=500):
+
+        # Run the task once to ensure we catch silly errors early
+        task(self)
+
+        msg_id_send = messaging.MSG_TMP_BASE + 100
+        msg_id_recv = 0
+
+        got_work_done = {"count": 0, "errors": 0}
+
+        def work_done_callback(got_work_done, msg_type, src, failure):
+            got_work_done["count"] += 1
+            if failure != "":
+                got_work_done["errors"] += 1
+                print failure + "\n"
+            os.kill(src.pid, signal.SIGTERM)
+
+        master_ctx = self.get_context((1,))
+        msg_id_done = master_ctx.register((work_done_callback, got_work_done))
+
+        pids = []
+
+        for y in range(workers):
+
+            pid = os.fork()
+            if pid == 0:
+                worker_ctx = self.get_context((1,))
+
+                failed = False
+                for x in range(num_ops / workers):
+                    try:
+                        task(self)
+                    except Exception as err:
+                        failed = True
+                        worker_ctx.send(master_ctx.server_id, msg_id_done, str(err))
+                        break
+
+                if failed == False:
+                    worker_ctx.send(master_ctx.server_id, msg_id_done, "")
+
+                worker_ctx.loop_once(60)
+                os._exit(0)
+            else:
+                pids.append(pid)
+
+            # Wait a little after each fork and run the event loop
+            timeout = False
+            start_time = time.time()
+            while (got_work_done["count"] < workers) and not timeout:
+                master_ctx.loop_once(0.1)
+                if time.time() - start_time > 0.5:
+                    timeout = True
+
+        timeout = False
+        start_time = time.time()
+
+        while (got_work_done["count"] < workers) and not timeout:
+            master_ctx.loop_once(0.1)
+            if time.time() - start_time > num_ops:
+                timeout = True
+
+        for pid in pids:
+            if timeout:
+                os.kill(pid, signal.SIGTERM)
+            os.waitpid(pid, 0)
+
+        self.assertFalse(timeout, "Timeout waiting for child tasks to exit")
+        self.assertEquals(got_work_done["errors"], 0)
diff --git a/source4/dsdb/tests/python/ad_dc_multi_search.py b/source4/dsdb/tests/python/ad_dc_multi_search.py
new file mode 100644
index 00000000000..14cac0f58b7
--- /dev/null
+++ b/source4/dsdb/tests/python/ad_dc_multi_search.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import optparse
+import sys
+sys.path.insert(0, 'bin/python')
+
+import os
+import samba
+import samba.getopt as options
+import random
+import tempfile
+import shutil
+import time
+
+
+# We try to use the test infrastructure of Samba 4.3+, but if it
+# doesn't work, we are probably in a back-ported patch and trying to
+# run on 4.1 or something.
+#
+# Don't copy this horror into ordinary tests -- it is special for
+# performance tests that want to apply to old versions.
+try:
+    from samba.tests.subunitrun import SubunitOptions, TestProgram
+    ANCIENT_SAMBA = False
+except ImportError:
+    ANCIENT_SAMBA = True
+    samba.ensure_external_module("testtools", "testtools")
+    samba.ensure_external_module("subunit", "subunit/python")
+    from subunit.run import SubunitTestRunner
+    import unittest
+
+from samba.samdb import SamDB
+from samba.auth import system_session
+from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
+import ad_dc_multi_ops
+parser = optparse.OptionParser("ad_dc_mulit_bind.py [options] <host>")
+sambaopts = options.SambaOptions(parser)
+parser.add_option_group(sambaopts)
+parser.add_option_group(options.VersionOptions(parser))
+
+if not ANCIENT_SAMBA:
+    subunitopts = SubunitOptions(parser)
+    parser.add_option_group(subunitopts)
+
+# use command line creds if available
+credopts = options.CredentialsOptions(parser)
+parser.add_option_group(credopts)
+opts, args = parser.parse_args()
+
+if len(args) < 1:
+    parser.print_usage()
+    sys.exit(1)
+
+host = args[0]
+
+lp = sambaopts.get_loadparm()
+creds = credopts.get_credentials(lp)
+
+from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
+
+import ad_dc_multi_ops
+class MultiSearchTests(ad_dc_multi_ops.MultiOpsTests):
+    def setUp(self):
+        super(MultiSearchTests, self).setUp()
+        self.lp = lp
+
+    def test_250_concurrent_subtree_search_1_worker(self):
+        def subtree_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(subtree_search, 1, num_ops=250)
+
+
+    def test_500_concurrent_unindexed_search_1_workers(self):
+        def unindexed_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         expression="samAccountName=administrator*",
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(unindexed_search, 1)
+
+
+    def test_250_concurrent_subtree_search_2_workers(self):
+        def subtree_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(subtree_search, 2, num_ops=250)
+
+
+    def test_500_concurrent_unindexed_search_2_workers(self):
+        def unindexed_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         expression="samAccountName=administrator*",
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(unindexed_search, 2)
+
+
+    def test_250_concurrent_subtree_search_5_workers(self):
+        def subtree_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(subtree_search, 5, num_ops=250)
+
+
+    def test_500_concurrent_unindexed_search_5_workers(self):
+        def unindexed_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         expression="samAccountName=administrator*",
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(unindexed_search, 5)
+
+
+    def test_250_concurrent_subtree_search_10_workers(self):
+        def subtree_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(subtree_search, 10, num_ops=250)
+
+
+    def test_500_concurrent_unindexed_search_10_workers(self):
+        def unindexed_search(self):
+            samdb = SamDB(host, credentials=creds,
+                          session_info=system_session(self.lp), lp=self.lp)
+            samdb.search(base=samdb.domain_dn(),
+                         expression="samAccountName=administrator*",
+                         scope=SCOPE_SUBTREE, attrs=["*"])
+
+        self._test_concurrent_things(unindexed_search, 10)
+
+
+if "://" not in host:
+    if os.path.isfile(host):
+        host = "tdb://%s" % host
+    else:
+        host = "ldap://%s"; % host
+
+
+if ANCIENT_SAMBA:
+    runner = SubunitTestRunner()
+    if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
+        sys.exit(1)
+    sys.exit(0)
+else:
+    TestProgram(module=__name__, opts=subunitopts)
-- 
2.11.0


From 31be67221c35b0e96517423bfa2dd8d4d4e00659 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Mon, 17 Oct 2016 13:55:42 +1300
Subject: [PATCH 02/19] ldap: Run the LDAP server with the default (typically
 standard) process model

This allows one LDAP socket to proceed if another fails, and reduces the
impact of a crash becoming a DoS bug, as it only impacts one socket.

This may mean we have a lot of idle tasks, but this should not be a big
issue

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 source4/ldap_server/ldap_server.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/source4/ldap_server/ldap_server.c b/source4/ldap_server/ldap_server.c
index 96df39c5313..42bdb4b0e56 100644
--- a/source4/ldap_server/ldap_server.c
+++ b/source4/ldap_server/ldap_server.c
@@ -1053,9 +1053,12 @@ static void ldapsrv_task_init(struct task_server *task)
 
 	task_server_set_title(task, "task[ldapsrv]");
 
-	/* run the ldap server as a single process */
-	model_ops = process_model_startup("single");
-	if (!model_ops) goto failed;
+	/*
+	 * Here we used to run the ldap server as a single process,
+	 * but we don't want transaction locks for one task in a write
+	 * blocking all other reads, so we go multi-process.
+	 */
+	model_ops = task->model_ops;
 
 	ldap_service = talloc_zero(task, struct ldapsrv_service);
 	if (ldap_service == NULL) goto failed;
-- 
2.11.0


From ada5aaf0e63b6495f4d28de1932057fb63874d9f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 19:11:06 +1300
Subject: [PATCH 03/19] tdb: Improve debugging when the allrecord lock fails to
 upgrade

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/tdb/common/lock.c        |  2 ++
 lib/tdb/common/transaction.c | 18 +++++++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/tdb/common/lock.c b/lib/tdb/common/lock.c
index 195dbb5daf2..7dd76beb0d9 100644
--- a/lib/tdb/common/lock.c
+++ b/lib/tdb/common/lock.c
@@ -257,12 +257,14 @@ int tdb_allrecord_upgrade(struct tdb_context *tdb)
 		TDB_LOG((tdb, TDB_DEBUG_ERROR,
 			 "tdb_allrecord_upgrade failed: count %u too high\n",
 			 tdb->allrecord_lock.count));
+		tdb->ecode = TDB_ERR_LOCK;
 		return -1;
 	}
 
 	if (tdb->allrecord_lock.off != 1) {
 		TDB_LOG((tdb, TDB_DEBUG_ERROR,
 			 "tdb_allrecord_upgrade failed: already upgraded?\n"));
+		tdb->ecode = TDB_ERR_LOCK;
 		return -1;
 	}
 
diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
index 0dd057ba7a7..e22aada4cbf 100644
--- a/lib/tdb/common/transaction.c
+++ b/lib/tdb/common/transaction.c
@@ -982,7 +982,23 @@ static int _tdb_transaction_prepare_commit(struct tdb_context *tdb)
 
 	/* upgrade the main transaction lock region to a write lock */
 	if (tdb_allrecord_upgrade(tdb) == -1) {
-		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_transaction_prepare_commit: failed to upgrade hash locks\n"));
+		if (tdb->ecode == TDB_ERR_RDONLY && tdb->read_only) {
+			TDB_LOG((tdb, TDB_DEBUG_ERROR,
+				 "tdb_transaction_prepare_commit: "
+				 "failed to upgrade hash locks: "
+				 "database is read only\n"));
+		} else if (tdb->ecode == TDB_ERR_RDONLY
+			   && tdb->traverse_read) {
+			TDB_LOG((tdb, TDB_DEBUG_ERROR,
+				 "tdb_transaction_prepare_commit: "
+				 "failed to upgrade hash locks: "
+				 "a database traverse is in progress\n"));
+		} else {
+			TDB_LOG((tdb, TDB_DEBUG_ERROR,
+				 "tdb_transaction_prepare_commit: "
+				 "failed to upgrade hash locks: %s\n",
+				 tdb_errorstr(tdb)));
+		}
 		_tdb_transaction_cancel(tdb);
 		return -1;
 	}
-- 
2.11.0


From c904cae1635f1be7730f1579fe03aa17c624121c Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Fri, 31 Mar 2017 17:34:13 +1300
Subject: [PATCH 04/19] tdb: Remove locking from tdb_traverse_read()

This restores the original intent of tdb_traverse_read() in
7dd31288a701d772e45b1960ac4ce4cc1be782ed

This is needed to avoid a deadlock with tdb_lockall() and the
transaction start, as ldb_tdb should take the allrecord lock during a
search (which calls tdb_traverse), and can otherwise deadlock against
a transaction starting in another process

We add a test to show that a transaction can now start while a read
traverse is in progress

This allows more operations to happen in parallel.  The blocking point
is moved to the prepare commit.

This in turn permits a roughly doubling of unindexed search
performance, because currently ldb_tdb omits to take the lock due to
an unrelated bug, but taking the allrecord lock triggers the
above-mentioned deadlock.

This behaviour was added in 251aaafe3a9213118ac3a92def9ab2104c40d12a
for Solaris 10 in 2005.

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/tdb/common/traverse.c          | 10 +---------
 lib/tdb/test/run-nested-traverse.c | 31 +++++++++++++++++++++++++++----
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/lib/tdb/common/traverse.c b/lib/tdb/common/traverse.c
index f33ef34ab8d..f62306e5560 100644
--- a/lib/tdb/common/traverse.c
+++ b/lib/tdb/common/traverse.c
@@ -244,7 +244,7 @@ out:
 
 
 /*
-  a read style traverse - temporarily marks the db read only
+  a read style traverse - temporarily marks each record read only
 */
 _PUBLIC_ int tdb_traverse_read(struct tdb_context *tdb,
 		      tdb_traverse_func fn, void *private_data)
@@ -252,19 +252,11 @@ _PUBLIC_ int tdb_traverse_read(struct tdb_context *tdb,
 	struct tdb_traverse_lock tl = { NULL, 0, 0, F_RDLCK };
 	int ret;
 
-	/* we need to get a read lock on the transaction lock here to
-	   cope with the lock ordering semantics of solaris10 */
-	if (tdb_transaction_lock(tdb, F_RDLCK, TDB_LOCK_WAIT)) {
-		return -1;
-	}
-
 	tdb->traverse_read++;
 	tdb_trace(tdb, "tdb_traverse_read_start");
 	ret = tdb_traverse_internal(tdb, fn, private_data, &tl);
 	tdb->traverse_read--;
 
-	tdb_transaction_unlock(tdb, F_RDLCK);
-
 	return ret;
 }
 
diff --git a/lib/tdb/test/run-nested-traverse.c b/lib/tdb/test/run-nested-traverse.c
index 22ee3e2a2a6..aeaa0859793 100644
--- a/lib/tdb/test/run-nested-traverse.c
+++ b/lib/tdb/test/run-nested-traverse.c
@@ -41,7 +41,30 @@ static int traverse2(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
 	return 0;
 }
 
-static int traverse1(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+static int traverse1r(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+		     void *p)
+{
+	ok1(correct_key(key));
+	ok1(correct_data(data));
+	ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
+	    == SUCCESS);
+	ok1(external_agent_operation(agent, STORE, tdb_name(tdb))
+	    == SUCCESS);
+	ok1(external_agent_operation(agent, TRANSACTION_COMMIT, tdb_name(tdb))
+	    == WOULD_HAVE_BLOCKED);
+	tdb_traverse(tdb, traverse2, NULL);
+
+	/* That should *not* release the all-records lock! */
+	ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
+	    == SUCCESS);
+	ok1(external_agent_operation(agent, STORE, tdb_name(tdb))
+	    == SUCCESS);
+	ok1(external_agent_operation(agent, TRANSACTION_COMMIT, tdb_name(tdb))
+	    == WOULD_HAVE_BLOCKED);
+	return 0;
+}
+
+static int traverse1w(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
 		     void *p)
 {
 	ok1(correct_key(key));
@@ -50,7 +73,7 @@ static int traverse1(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
 	    == WOULD_HAVE_BLOCKED);
 	tdb_traverse(tdb, traverse2, NULL);
 
-	/* That should *not* release the transaction lock! */
+	/* That should *not* release the all-records lock! */
 	ok1(external_agent_operation(agent, TRANSACTION_START, tdb_name(tdb))
 	    == WOULD_HAVE_BLOCKED);
 	return 0;
@@ -80,8 +103,8 @@ int main(int argc, char *argv[])
 	data.dsize = strlen("world");
 
 	ok1(tdb_store(tdb, key, data, TDB_INSERT) == 0);
-	tdb_traverse(tdb, traverse1, NULL);
-	tdb_traverse_read(tdb, traverse1, NULL);
+	tdb_traverse(tdb, traverse1w, NULL);
+	tdb_traverse_read(tdb, traverse1r, NULL);
 	tdb_close(tdb);
 
 	return exit_status();
-- 
2.11.0


From b2cd4fa57dd4a2e2d9c5a2fa71bf74ba0c289407 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Fri, 31 Mar 2017 17:35:06 +1300
Subject: [PATCH 05/19] tdb: Improve debugging in _tdb_transaction_start

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/tdb/common/transaction.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/tdb/common/transaction.c b/lib/tdb/common/transaction.c
index e22aada4cbf..420e754d9b6 100644
--- a/lib/tdb/common/transaction.c
+++ b/lib/tdb/common/transaction.c
@@ -476,6 +476,10 @@ static int _tdb_transaction_start(struct tdb_context *tdb,
 		SAFE_FREE(tdb->transaction);
 		if ((lockflags & TDB_LOCK_WAIT) == 0) {
 			tdb->ecode = TDB_ERR_NOLOCK;
+		} else {
+			TDB_LOG((tdb, TDB_DEBUG_ERROR,
+				 "tdb_transaction_start: "
+				 "failed to get transaction lock\n"));
 		}
 		return -1;
 	}
-- 
2.11.0


From 888611daa961f876aa225cd776ea1bd08f91c161 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Mon, 3 Apr 2017 11:20:10 +1200
Subject: [PATCH 06/19] tdb: Release version 1.3.13

This version of TDB removes the protection between tdb_traverse_read()
and tdb_transaction_start(), allowing Samba's AD DC (on ldb_tdb) to
address a failure to take the all-record lock.  This in turn permits a
roughly doubling of unindexed search performance.

The downside is that this change was made to support Solaris 10.  However
little is known about what exactly triggered this requirement and
Solaris 10 is not as common now as it was a decade ago.

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/tdb/ABI/tdb-1.3.13.sigs | 70 +++++++++++++++++++++++++++++++++++++++++++++
 lib/tdb/wscript             |  2 +-
 2 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 lib/tdb/ABI/tdb-1.3.13.sigs

diff --git a/lib/tdb/ABI/tdb-1.3.13.sigs b/lib/tdb/ABI/tdb-1.3.13.sigs
new file mode 100644
index 00000000000..48f4278890a
--- /dev/null
+++ b/lib/tdb/ABI/tdb-1.3.13.sigs
@@ -0,0 +1,70 @@
+tdb_add_flags: void (struct tdb_context *, unsigned int)
+tdb_append: int (struct tdb_context *, TDB_DATA, TDB_DATA)
+tdb_chainlock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_mark: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_nonblock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_read: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_read_nonblock: int (struct tdb_context *, TDB_DATA)
+tdb_chainlock_unmark: int (struct tdb_context *, TDB_DATA)
+tdb_chainunlock: int (struct tdb_context *, TDB_DATA)
+tdb_chainunlock_read: int (struct tdb_context *, TDB_DATA)
+tdb_check: int (struct tdb_context *, int (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_close: int (struct tdb_context *)
+tdb_delete: int (struct tdb_context *, TDB_DATA)
+tdb_dump_all: void (struct tdb_context *)
+tdb_enable_seqnum: void (struct tdb_context *)
+tdb_error: enum TDB_ERROR (struct tdb_context *)
+tdb_errorstr: const char *(struct tdb_context *)
+tdb_exists: int (struct tdb_context *, TDB_DATA)
+tdb_fd: int (struct tdb_context *)
+tdb_fetch: TDB_DATA (struct tdb_context *, TDB_DATA)
+tdb_firstkey: TDB_DATA (struct tdb_context *)
+tdb_freelist_size: int (struct tdb_context *)
+tdb_get_flags: int (struct tdb_context *)
+tdb_get_logging_private: void *(struct tdb_context *)
+tdb_get_seqnum: int (struct tdb_context *)
+tdb_hash_size: int (struct tdb_context *)
+tdb_increment_seqnum_nonblock: void (struct tdb_context *)
+tdb_jenkins_hash: unsigned int (TDB_DATA *)
+tdb_lock_nonblock: int (struct tdb_context *, int, int)
+tdb_lockall: int (struct tdb_context *)
+tdb_lockall_mark: int (struct tdb_context *)
+tdb_lockall_nonblock: int (struct tdb_context *)
+tdb_lockall_read: int (struct tdb_context *)
+tdb_lockall_read_nonblock: int (struct tdb_context *)
+tdb_lockall_unmark: int (struct tdb_context *)
+tdb_log_fn: tdb_log_func (struct tdb_context *)
+tdb_map_size: size_t (struct tdb_context *)
+tdb_name: const char *(struct tdb_context *)
+tdb_nextkey: TDB_DATA (struct tdb_context *, TDB_DATA)
+tdb_null: dptr = 0xXXXX, dsize = 0
+tdb_open: struct tdb_context *(const char *, int, int, int, mode_t)
+tdb_open_ex: struct tdb_context *(const char *, int, int, int, mode_t, const struct tdb_logging_context *, tdb_hash_func)
+tdb_parse_record: int (struct tdb_context *, TDB_DATA, int (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_printfreelist: int (struct tdb_context *)
+tdb_remove_flags: void (struct tdb_context *, unsigned int)
+tdb_reopen: int (struct tdb_context *)
+tdb_reopen_all: int (int)
+tdb_repack: int (struct tdb_context *)
+tdb_rescue: int (struct tdb_context *, void (*)(TDB_DATA, TDB_DATA, void *), void *)
+tdb_runtime_check_for_robust_mutexes: bool (void)
+tdb_set_logging_function: void (struct tdb_context *, const struct tdb_logging_context *)
+tdb_set_max_dead: void (struct tdb_context *, int)
+tdb_setalarm_sigptr: void (struct tdb_context *, volatile sig_atomic_t *)
+tdb_store: int (struct tdb_context *, TDB_DATA, TDB_DATA, int)
+tdb_storev: int (struct tdb_context *, TDB_DATA, const TDB_DATA *, int, int)
+tdb_summary: char *(struct tdb_context *)
+tdb_transaction_cancel: int (struct tdb_context *)
+tdb_transaction_commit: int (struct tdb_context *)
+tdb_transaction_prepare_commit: int (struct tdb_context *)
+tdb_transaction_start: int (struct tdb_context *)
+tdb_transaction_start_nonblock: int (struct tdb_context *)
+tdb_transaction_write_lock_mark: int (struct tdb_context *)
+tdb_transaction_write_lock_unmark: int (struct tdb_context *)
+tdb_traverse: int (struct tdb_context *, tdb_traverse_func, void *)
+tdb_traverse_read: int (struct tdb_context *, tdb_traverse_func, void *)
+tdb_unlock: int (struct tdb_context *, int, int)
+tdb_unlockall: int (struct tdb_context *)
+tdb_unlockall_read: int (struct tdb_context *)
+tdb_validate_freelist: int (struct tdb_context *, int *)
+tdb_wipe_all: int (struct tdb_context *)
diff --git a/lib/tdb/wscript b/lib/tdb/wscript
index 0d682ebb4cd..4f8439d76ef 100644
--- a/lib/tdb/wscript
+++ b/lib/tdb/wscript
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 APPNAME = 'tdb'
-VERSION = '1.3.12'
+VERSION = '1.3.13'
 
 blddir = 'bin'
 
-- 
2.11.0


From bbadaf368d06584d1c259ce44cc09337fd2239eb Mon Sep 17 00:00:00 2001
From: Garming Sam <garming@xxxxxxxxxxxxxxx>
Date: Thu, 30 Mar 2017 12:03:17 +1300
Subject: [PATCH 07/19] ldb_tdb: Ensure we correctly decrement
 ltdb->read_lock_count

If we do not do this, then we never take the all record lock, and instead do a lock
for every record as we go, which is very slow during a large search

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
Signed-off-by: Garming Sam <garming@xxxxxxxxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 6b1187e64ad..7febf0720be 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -119,6 +119,7 @@ int ltdb_unlock_read(struct ldb_module *module)
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	if (ltdb->in_transaction == 0 && ltdb->read_lock_count == 1) {
 		tdb_unlockall_read(ltdb->tdb);
+		ltdb->read_lock_count--;
 		return 0;
 	}
 	ltdb->read_lock_count--;
-- 
2.11.0


From 37b2e2b1a8f274accef2b20d83acbde7207c689f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 14:26:23 +1300
Subject: [PATCH 08/19] ldb_tdb: Provide better debugging on prepare_commit
 failures

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 7febf0720be..8b2f09904c0 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1124,6 +1124,7 @@ static int ltdb_start_trans(struct ldb_module *module)
 
 static int ltdb_prepare_commit(struct ldb_module *module)
 {
+	int ret;
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
@@ -1131,15 +1132,21 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 		return LDB_SUCCESS;
 	}
 
-	if (ltdb_index_transaction_commit(module) != 0) {
+	ret = ltdb_index_transaction_commit(module);
+	if (ret != LDB_SUCCESS) {
 		tdb_transaction_cancel(ltdb->tdb);
 		ltdb->in_transaction--;
-		return ltdb_err_map(tdb_error(ltdb->tdb));
+		return ret;
 	}
 
 	if (tdb_transaction_prepare_commit(ltdb->tdb) != 0) {
+		ret = ltdb_err_map(tdb_error(ltdb->tdb));
 		ltdb->in_transaction--;
-		return ltdb_err_map(tdb_error(ltdb->tdb));
+		ldb_asprintf_errstring(ldb_module_get_ctx(module),
+				       "Failure during tdb_transaction_prepare_commit(): %s -> %s",
+				       tdb_errorstr(ltdb->tdb),
+				       ldb_strerror(ret));
+		return ret;
 	}
 
 	ltdb->prepared_commit = true;
-- 
2.11.0


From fe91fc3b822b5dba33faa58eabe157a6df5f628e Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 14:27:55 +1300
Subject: [PATCH 09/19] ldb_tdb: Provide better debugging on end_trans failures

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_tdb.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 8b2f09904c0..58109d05c97 100644
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -1156,11 +1156,12 @@ static int ltdb_prepare_commit(struct ldb_module *module)
 
 static int ltdb_end_trans(struct ldb_module *module)
 {
+	int ret;
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 
 	if (!ltdb->prepared_commit) {
-		int ret = ltdb_prepare_commit(module);
+		ret = ltdb_prepare_commit(module);
 		if (ret != LDB_SUCCESS) {
 			return ret;
 		}
@@ -1170,7 +1171,12 @@ static int ltdb_end_trans(struct ldb_module *module)
 	ltdb->prepared_commit = false;
 
 	if (tdb_transaction_commit(ltdb->tdb) != 0) {
-		return ltdb_err_map(tdb_error(ltdb->tdb));
+		ret = ltdb_err_map(tdb_error(ltdb->tdb));
+		ldb_asprintf_errstring(ldb_module_get_ctx(module),
+				       "Failure during tdb_transaction_commit(): %s -> %s",
+				       tdb_errorstr(ltdb->tdb),
+				       ldb_strerror(ret));
+		return ret;
 	}
 
 	return LDB_SUCCESS;
-- 
2.11.0


From 53360ee3a16522bdebb3b6c1d33a86016e13905d Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:10:08 +1300
Subject: [PATCH 10/19] ldb_tdb: Split index load out into a sub-funciton:
 ltdb_index_load

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_cache.c | 73 +++++++++++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 5ea09d6c7ea..8928ccaf274 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -226,6 +226,49 @@ failed:
 	return -1;
 }
 
+/*
+  register any index records we find for the DB
+*/
+static int ltdb_index_load(struct ldb_module *module,
+			   struct ltdb_private *ltdb)
+{
+	struct ldb_dn *indexlist_dn;
+	int r;
+
+	struct ldb_context *ldb = ldb_module_get_ctx(module);
+
+	talloc_free(ltdb->cache->indexlist);
+
+	ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
+	if (ltdb->cache->indexlist == NULL) {
+		return -1;
+	}
+	ltdb->cache->one_level_indexes = false;
+	ltdb->cache->attribute_indexes = false;
+
+	indexlist_dn = ldb_dn_new(ltdb, ldb, LTDB_INDEXLIST);
+	if (indexlist_dn == NULL) {
+		return -1;
+	}
+
+	r = ltdb_search_dn1(module, indexlist_dn, ltdb->cache->indexlist,
+			    LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
+			    |LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC
+			    |LDB_UNPACK_DATA_FLAG_NO_DN);
+	TALLOC_FREE(indexlist_dn);
+
+	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
+		return -1;
+	}
+
+	if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXONE) != NULL) {
+		ltdb->cache->one_level_indexes = true;
+	}
+	if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
+		ltdb->cache->attribute_indexes = true;
+	}
+	return 0;
+}
 
 /*
   initialise the baseinfo record
@@ -316,7 +359,6 @@ int ltdb_cache_load(struct ldb_module *module)
 	void *data = ldb_module_get_private(module);
 	struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
 	struct ldb_dn *baseinfo_dn = NULL, *options_dn = NULL;
-	struct ldb_dn *indexlist_dn = NULL;
 	uint64_t seq;
 	struct ldb_message *baseinfo = NULL, *options = NULL;
 	int r;
@@ -332,10 +374,6 @@ int ltdb_cache_load(struct ldb_module *module)
 	if (ltdb->cache == NULL) {
 		ltdb->cache = talloc_zero(ltdb, struct ltdb_cache);
 		if (ltdb->cache == NULL) goto failed;
-		ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
-		if (ltdb->cache->indexlist == NULL) {
-			goto failed;
-		}
 	}
 
 	baseinfo = ldb_msg_new(ltdb->cache);
@@ -402,7 +440,6 @@ int ltdb_cache_load(struct ldb_module *module)
 		ltdb->disallow_dn_filter = false;
 	}
 
-	talloc_free(ltdb->cache->indexlist);
 	/*
 	 * ltdb_attributes_unload() calls internally talloc_free() on
 	 * any non-fixed elemnts in ldb->schema.attributes.
@@ -412,31 +449,11 @@ int ltdb_cache_load(struct ldb_module *module)
 	 * partition module.
 	 */
 	ltdb_attributes_unload(module);
-	ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
-	if (ltdb->cache->indexlist == NULL) {
-		goto failed;
-	}
-	ltdb->cache->one_level_indexes = false;
-	ltdb->cache->attribute_indexes = false;
-	    
-	indexlist_dn = ldb_dn_new(module, ldb, LTDB_INDEXLIST);
-	if (indexlist_dn == NULL) goto failed;
 
-	r = ltdb_search_dn1(module, indexlist_dn, ltdb->cache->indexlist,
-			    LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC
-			    |LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC
-			    |LDB_UNPACK_DATA_FLAG_NO_DN);
-	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
+	if (ltdb_index_load(module, ltdb) == -1) {
 		goto failed;
 	}
 
-	if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXONE) != NULL) {
-		ltdb->cache->one_level_indexes = true;
-	}
-	if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
-		ltdb->cache->attribute_indexes = true;
-	}
-
 	/*
 	 * NOTE WELL: This is per-ldb, not per module, so overwrites
 	 * the handlers across all databases when used under Samba's
@@ -449,13 +466,11 @@ int ltdb_cache_load(struct ldb_module *module)
 done:
 	talloc_free(options);
 	talloc_free(baseinfo);
-	talloc_free(indexlist_dn);
 	return 0;
 
 failed:
 	talloc_free(options);
 	talloc_free(baseinfo);
-	talloc_free(indexlist_dn);
 	return -1;
 }
 
-- 
2.11.0


From b1662cf53027a506ba2795d4a502b8244af46cb8 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:07:16 +1300
Subject: [PATCH 11/19] ldb: change the arguments to ldb_is_indexed() to
 provide the ltdb_private

By doing this, we can be more efficient in locating if we have an index in
the future.

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_index.c | 58 ++++++++++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 53fcde5a31d..8912aea9f3c 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -445,12 +445,16 @@ static struct ldb_dn *ltdb_index_key(struct ldb_context *ldb,
 /*
   see if a attribute value is in the list of indexed attributes
 */
-static bool ltdb_is_indexed(const struct ldb_message *index_list, const char *attr)
+static bool ltdb_is_indexed(struct ldb_module *module,
+			    struct ltdb_private *ltdb,
+			    const char *attr)
 {
 	unsigned int i;
 	struct ldb_message_element *el;
+	struct ldb_context *ldb = ldb_module_get_ctx(module);
+
 
-	el = ldb_msg_find_element(index_list, LTDB_IDXATTR);
+	el = ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR);
 	if (el == NULL) {
 		return false;
 	}
@@ -481,8 +485,8 @@ static bool ltdb_is_indexed(const struct ldb_message *index_list, const char *at
   equality search only)
  */
 static int ltdb_index_dn_simple(struct ldb_module *module,
+				struct ltdb_private *ltdb,
 				const struct ldb_parse_tree *tree,
-				const struct ldb_message *index_list,
 				struct dn_list *list)
 {
 	struct ldb_context *ldb;
@@ -496,7 +500,7 @@ static int ltdb_index_dn_simple(struct ldb_module *module,
 
 	/* if the attribute isn't in the list of indexed attributes then
 	   this node needs a full search */
-	if (!ltdb_is_indexed(index_list, tree->u.equality.attr)) {
+	if (!ltdb_is_indexed(module, ltdb, tree->u.equality.attr)) {
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
@@ -517,12 +521,10 @@ static bool list_union(struct ldb_context *, struct dn_list *, const struct dn_l
   return a list of dn's that might match a leaf indexed search
  */
 static int ltdb_index_dn_leaf(struct ldb_module *module,
+			      struct ltdb_private *ltdb,
 			      const struct ldb_parse_tree *tree,
-			      const struct ldb_message *index_list,
 			      struct dn_list *list)
 {
-	struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module),
-						    struct ltdb_private);
 	if (ltdb->disallow_dn_filter &&
 	    (ldb_attr_cmp(tree->u.equality.attr, "dn") == 0)) {
 		/* in AD mode we do not support "(dn=...)" search filters */
@@ -540,7 +542,7 @@ static int ltdb_index_dn_leaf(struct ldb_module *module,
 		list->count = 1;
 		return LDB_SUCCESS;
 	}
-	return ltdb_index_dn_simple(module, tree, index_list, list);
+	return ltdb_index_dn_simple(module, ltdb, tree, list);
 }
 
 
@@ -653,8 +655,8 @@ static bool list_union(struct ldb_context *ldb,
 }
 
 static int ltdb_index_dn(struct ldb_module *module,
+			 struct ltdb_private *ltdb,
 			 const struct ldb_parse_tree *tree,
-			 const struct ldb_message *index_list,
 			 struct dn_list *list);
 
 
@@ -662,8 +664,8 @@ static int ltdb_index_dn(struct ldb_module *module,
   process an OR list (a union)
  */
 static int ltdb_index_dn_or(struct ldb_module *module,
+			    struct ltdb_private *ltdb,
 			    const struct ldb_parse_tree *tree,
-			    const struct ldb_message *index_list,
 			    struct dn_list *list)
 {
 	struct ldb_context *ldb;
@@ -683,7 +685,8 @@ static int ltdb_index_dn_or(struct ldb_module *module,
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
 
-		ret = ltdb_index_dn(module, tree->u.list.elements[i], index_list, list2);
+		ret = ltdb_index_dn(module, ltdb,
+				    tree->u.list.elements[i], list2);
 
 		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 			/* X || 0 == X */
@@ -715,8 +718,8 @@ static int ltdb_index_dn_or(struct ldb_module *module,
   NOT an index results
  */
 static int ltdb_index_dn_not(struct ldb_module *module,
+			     struct ltdb_private *ltdb,
 			     const struct ldb_parse_tree *tree,
-			     const struct ldb_message *index_list,
 			     struct dn_list *list)
 {
 	/* the only way to do an indexed not would be if we could
@@ -746,8 +749,8 @@ static bool ltdb_index_unique(struct ldb_context *ldb,
   process an AND expression (intersection)
  */
 static int ltdb_index_dn_and(struct ldb_module *module,
+			     struct ltdb_private *ltdb,
 			     const struct ldb_parse_tree *tree,
-			     const struct ldb_message *index_list,
 			     struct dn_list *list)
 {
 	struct ldb_context *ldb;
@@ -771,7 +774,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
 			continue;
 		}
 
-		ret = ltdb_index_dn(module, subtree, index_list, list);
+		ret = ltdb_index_dn(module, ltdb,
+				    subtree, list);
 		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 			/* 0 && X == 0 */
 			return LDB_ERR_NO_SUCH_OBJECT;
@@ -798,7 +802,8 @@ static int ltdb_index_dn_and(struct ldb_module *module,
 			return ldb_module_oom(module);
 		}
 
-		ret = ltdb_index_dn(module, subtree, index_list, list2);
+		ret = ltdb_index_dn(module, ltdb,
+				    subtree, list2);
 
 		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 			/* X && 0 == 0 */
@@ -884,27 +889,27 @@ static int ltdb_index_dn_one(struct ldb_module *module,
   an error. return LDB_ERR_NO_SUCH_OBJECT for no matches, or LDB_SUCCESS for matches
  */
 static int ltdb_index_dn(struct ldb_module *module,
+			 struct ltdb_private *ltdb,
 			 const struct ldb_parse_tree *tree,
-			 const struct ldb_message *index_list,
 			 struct dn_list *list)
 {
 	int ret = LDB_ERR_OPERATIONS_ERROR;
 
 	switch (tree->operation) {
 	case LDB_OP_AND:
-		ret = ltdb_index_dn_and(module, tree, index_list, list);
+		ret = ltdb_index_dn_and(module, ltdb, tree, list);
 		break;
 
 	case LDB_OP_OR:
-		ret = ltdb_index_dn_or(module, tree, index_list, list);
+		ret = ltdb_index_dn_or(module, ltdb, tree, list);
 		break;
 
 	case LDB_OP_NOT:
-		ret = ltdb_index_dn_not(module, tree, index_list, list);
+		ret = ltdb_index_dn_not(module, ltdb, tree, list);
 		break;
 
 	case LDB_OP_EQUALITY:
-		ret = ltdb_index_dn_leaf(module, tree, index_list, list);
+		ret = ltdb_index_dn_leaf(module, ltdb, tree, list);
 		break;
 
 	case LDB_OP_SUBSTRING:
@@ -1087,7 +1092,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count)
 			talloc_free(dn_list);
 			return LDB_ERR_OPERATIONS_ERROR;
 		}
-		ret = ltdb_index_dn(ac->module, ac->tree, ltdb->cache->indexlist, dn_list);
+		ret = ltdb_index_dn(ac->module, ltdb, ac->tree, dn_list);
 		if (ret != LDB_SUCCESS) {
 			talloc_free(dn_list);
 			return ret;
@@ -1221,14 +1226,15 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn,
 		return LDB_SUCCESS;
 	}
 
-	if (ltdb->cache->indexlist->num_elements == 0) {
+	if (ltdb->cache->attribute_indexes == false) {
 		/* no indexed fields */
 		return LDB_SUCCESS;
 	}
 
 	for (i = 0; i < num_el; i++) {
 		int ret;
-		if (!ltdb_is_indexed(ltdb->cache->indexlist, elements[i].name)) {
+		if (!ltdb_is_indexed(module,
+				     ltdb, elements[i].name)) {
 			continue;
 		}
 		ret = ltdb_index_add_el(module, dn, &elements[i], is_new);
@@ -1306,7 +1312,8 @@ int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
 	if (ldb_dn_is_special(dn)) {
 		return LDB_SUCCESS;
 	}
-	if (!ltdb_is_indexed(ltdb->cache->indexlist, el->name)) {
+	if (!ltdb_is_indexed(module,
+			     ltdb, el->name)) {
 		return LDB_SUCCESS;
 	}
 	return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el, true);
@@ -1439,7 +1446,8 @@ int ltdb_index_del_element(struct ldb_module *module, struct ldb_dn *dn,
 		return LDB_SUCCESS;
 	}
 
-	if (!ltdb_is_indexed(ltdb->cache->indexlist, el->name)) {
+	if (!ltdb_is_indexed(module,
+			     ltdb, el->name)) {
 		return LDB_SUCCESS;
 	}
 	for (i = 0; i < el->num_values; i++) {
-- 
2.11.0


From 8f8217f3404d2262d2a42edf4d9a6b8a26afc0c0 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:21:34 +1300
Subject: [PATCH 12/19] ldb_tdb: consistently use
 ltdb->cache->attribute_indexes to determine if we have indexes

This is instead of checking the number of elements via ltdb->cache->indexlist->num_elements

In turn, this allows us to avoid fetching ltdb->cache->indexlist in the future

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_index.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index 8912aea9f3c..e121b8aeec5 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -453,6 +453,9 @@ static bool ltdb_is_indexed(struct ldb_module *module,
 	struct ldb_message_element *el;
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 
+	if (ltdb->cache->attribute_indexes == false) {
+		return false;
+	}
 
 	el = ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR);
 	if (el == NULL) {
@@ -1647,7 +1650,7 @@ int ltdb_reindex(struct ldb_module *module)
 	}
 
 	/* if we don't have indexes we have nothing todo */
-	if (ltdb->cache->indexlist->num_elements == 0) {
+	if (ltdb->cache->attribute_indexes == false) {
 		return LDB_SUCCESS;
 	}
 
-- 
2.11.0


From 7acf9ba9c238cdc4f5100facef8245bf591cb62f Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:23:44 +1300
Subject: [PATCH 13/19] ldb: Allow a caller (in particular Samba) to handle the
 list of attributes with an index

By doing that, Samba will use a binary search to locate the attributes
rather than an O(n) search, during every search or modify of the database.

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/common/ldb_attributes.c | 12 ++++++++++++
 lib/ldb/include/ldb.h           |  5 +++++
 lib/ldb/include/ldb_module.h    |  2 ++
 lib/ldb/include/ldb_private.h   |  7 +++++++
 lib/ldb/ldb_tdb/ldb_index.c     |  9 +++++++++
 5 files changed, 35 insertions(+)

diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c
index 2021a0bf64a..98ec5a409ba 100644
--- a/lib/ldb/common/ldb_attributes.c
+++ b/lib/ldb/common/ldb_attributes.c
@@ -383,3 +383,15 @@ void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
 	ldb->schema.attribute_handler_override_private = private_data;
 	ldb->schema.attribute_handler_override = override;
 }
+
+/*
+  set that the attribute handler override function - used to delegate
+  schema handling to external code, is handling setting
+  LDB_ATTR_FLAG_INDEXED
+ */
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+				       bool one_level_indexes)
+{
+	ldb->schema.index_handler_override = true;
+	ldb->schema.one_level_indexes = one_level_indexes;
+}
diff --git a/lib/ldb/include/ldb.h b/lib/ldb/include/ldb.h
index afcf9eb6351..a3babca08ed 100644
--- a/lib/ldb/include/ldb.h
+++ b/lib/ldb/include/ldb.h
@@ -441,6 +441,11 @@ const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_c
  */
 #define LDB_ATTR_FLAG_FROM_DB      (1<<6)
 
+/*
+ * The attribute was loaded from a DB, rather than via the C API
+ */
+#define LDB_ATTR_FLAG_INDEXED      (1<<7)
+
 /**
   LDAP attribute syntax for a DN
 
diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index 833d5a8f3d0..75f3fcb2bf8 100644
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -125,6 +125,8 @@ typedef const struct ldb_schema_attribute *(*ldb_attribute_handler_override_fn_t
 void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
 					       ldb_attribute_handler_override_fn_t override,
 					       void *private_data);
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+				       bool one_level_indexes);
 
 /* A useful function to build comparison functions with */
 int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx, 
diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h
index 644d49cd23f..bd975b81fec 100644
--- a/lib/ldb/include/ldb_private.h
+++ b/lib/ldb/include/ldb_private.h
@@ -88,6 +88,13 @@ struct ldb_schema {
 
 	unsigned num_dn_extended_syntax;
 	struct ldb_dn_extended_syntax *dn_extended_syntax;
+
+	/*
+	 * If set, the attribute_handler_override has the details of
+	 * what attributes have an index
+	 */
+	bool index_handler_override;
+	bool one_level_indexes;
 };
 
 /*
diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c
index e121b8aeec5..dd7c2db7829 100644
--- a/lib/ldb/ldb_tdb/ldb_index.c
+++ b/lib/ldb/ldb_tdb/ldb_index.c
@@ -453,6 +453,15 @@ static bool ltdb_is_indexed(struct ldb_module *module,
 	struct ldb_message_element *el;
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 
+	if (ldb->schema.index_handler_override) {
+		const struct ldb_schema_attribute *a
+			= ldb_schema_attribute_by_name(ldb, attr);
+		if (a->flags & LDB_ATTR_FLAG_INDEXED) {
+			return true;
+		} else {
+			return false;
+		}
+	}
 	if (ltdb->cache->attribute_indexes == false) {
 		return false;
 	}
-- 
2.11.0


From ebe8d309bf28478b163dd0272662e73240e297c7 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:10:22 +1300
Subject: [PATCH 14/19] ldb_tdb: Avoid reading the index list from the DB if we
 are already set to override it

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_cache.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 8928ccaf274..7b5e1fcf9d5 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -237,6 +237,14 @@ static int ltdb_index_load(struct ldb_module *module,
 
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
 
+	if (ldb->schema.index_handler_override) {
+		ltdb->cache->attribute_indexes = true;
+		ltdb->cache->one_level_indexes = ldb->schema.one_level_indexes;
+		/* we skip loading the @INDEXLIST record when a module is supplying
+		   its own attribute handling */
+		return 0;
+	}
+
 	talloc_free(ltdb->cache->indexlist);
 
 	ltdb->cache->indexlist = ldb_msg_new(ltdb->cache);
-- 
2.11.0


From 9bae78827d52175fcdf8c48a699d9c015c189c06 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:11:15 +1300
Subject: [PATCH 15/19] ldb_tdb: Call talloc_free(options_dn) as soon as we are
 done with options_dn

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ldb_tdb/ldb_cache.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c
index 7b5e1fcf9d5..dfef37e4bc8 100644
--- a/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/lib/ldb/ldb_tdb/ldb_cache.c
@@ -431,6 +431,7 @@ int ltdb_cache_load(struct ldb_module *module)
 	if (options_dn == NULL) goto failed;
 
 	r= ltdb_search_dn1(module, options_dn, options, 0);
+	talloc_free(options_dn);
 	if (r != LDB_SUCCESS && r != LDB_ERR_NO_SUCH_OBJECT) {
 		goto failed;
 	}
-- 
2.11.0


From d30ec7bc33b39a7413c8b69b48c8218d154ab5d1 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:54:58 +1300
Subject: [PATCH 16/19] ldb: Version 1.1.30

Including important fixes for ltdb locking and hooks for improved index performance in Samba

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 lib/ldb/ABI/ldb-1.1.30.sigs            | 269 +++++++++++++++++++++++++++++++++
 lib/ldb/ABI/pyldb-util-1.1.30.sigs     |   2 +
 lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs |   2 +
 lib/ldb/wscript                        |   2 +-
 4 files changed, 274 insertions(+), 1 deletion(-)
 create mode 100644 lib/ldb/ABI/ldb-1.1.30.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util-1.1.30.sigs
 create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs

diff --git a/lib/ldb/ABI/ldb-1.1.30.sigs b/lib/ldb/ABI/ldb-1.1.30.sigs
new file mode 100644
index 00000000000..9b865ed8907
--- /dev/null
+++ b/lib/ldb/ABI/ldb-1.1.30.sigs
@@ -0,0 +1,269 @@
+ldb_add: int (struct ldb_context *, const struct ldb_message *)
+ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *)
+ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...)
+ldb_attr_casefold: char *(TALLOC_CTX *, const char *)
+ldb_attr_dn: int (const char *)
+ldb_attr_in_list: int (const char * const *, const char *)
+ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *)
+ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *)
+ldb_base64_decode: int (char *)
+ldb_base64_encode: char *(TALLOC_CTX *, const char *, int)
+ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *)
+ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val)
+ldb_binary_encode_string: char *(TALLOC_CTX *, const char *)
+ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *)
+ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t)
+ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t)
+ldb_check_critical_controls: int (struct ldb_control **)
+ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *)
+ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **)
+ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *)
+ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *)
+ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_debug_add: void (struct ldb_context *, const char *, ...)
+ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level)
+ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...)
+ldb_delete: int (struct ldb_context *, struct ldb_dn *)
+ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...)
+ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *)
+ldb_dn_check_special: bool (struct ldb_dn *, const char *)
+ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val)
+ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *)
+ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *)
+ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *)
+ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *)
+ldb_dn_get_casefold: const char *(struct ldb_dn *)
+ldb_dn_get_comp_num: int (struct ldb_dn *)
+ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int)
+ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int)
+ldb_dn_get_extended_comp_num: int (struct ldb_dn *)
+ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *)
+ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int)
+ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *)
+ldb_dn_get_linearized: const char *(struct ldb_dn *)
+ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *)
+ldb_dn_get_rdn_name: const char *(struct ldb_dn *)
+ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *)
+ldb_dn_has_extended: bool (struct ldb_dn *)
+ldb_dn_is_null: bool (struct ldb_dn *)
+ldb_dn_is_special: bool (struct ldb_dn *)
+ldb_dn_is_valid: bool (struct ldb_dn *)
+ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *)
+ldb_dn_minimise: bool (struct ldb_dn *)
+ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *)
+ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...)
+ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int)
+ldb_dn_remove_extended_components: void (struct ldb_dn *)
+ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *)
+ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val)
+ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *)
+ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *)
+ldb_dn_validate: bool (struct ldb_dn *)
+ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *)
+ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int)
+ldb_errstring: const char *(struct ldb_context *)
+ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **)
+ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_create_perms: unsigned int (struct ldb_context *)
+ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_event_context: struct tevent_context *(struct ldb_context *)
+ldb_get_flags: unsigned int (struct ldb_context *)
+ldb_get_opaque: void *(struct ldb_context *, const char *)
+ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *)
+ldb_global_init: int (void)
+ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *)
+ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *)
+ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *)
+ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *)
+ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **)
+ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *)
+ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *)
+ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *)
+ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *)
+ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **)
+ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *)
+ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *)
+ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *)
+ldb_load_modules: int (struct ldb_context *, const char **)
+ldb_map_add: int (struct ldb_module *, struct ldb_request *)
+ldb_map_delete: int (struct ldb_module *, struct ldb_request *)
+ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *)
+ldb_map_modify: int (struct ldb_module *, struct ldb_request *)
+ldb_map_rename: int (struct ldb_module *, struct ldb_request *)
+ldb_map_search: int (struct ldb_module *, struct ldb_request *)
+ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope)
+ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *)
+ldb_match_msg_objectclass: int (const struct ldb_message *, const char *)
+ldb_mod_register_control: int (struct ldb_module *, const char *)
+ldb_modify: int (struct ldb_context *, const struct ldb_message *)
+ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *)
+ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **)
+ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int)
+ldb_module_flags: uint32_t (struct ldb_context *)
+ldb_module_get_ctx: struct ldb_context *(struct ldb_module *)
+ldb_module_get_name: const char *(struct ldb_module *)
+ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *)
+ldb_module_get_private: void *(struct ldb_module *)
+ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *)
+ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **)
+ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *)
+ldb_module_next: struct ldb_module *(struct ldb_module *)
+ldb_module_popt_options: struct poptOption **(struct ldb_context *)
+ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **)
+ldb_module_send_referral: int (struct ldb_request *, char *)
+ldb_module_set_next: void (struct ldb_module *, struct ldb_module *)
+ldb_module_set_private: void (struct ldb_module *, void *)
+ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type)
+ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_modules_load: int (const char *, const char *)
+ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int)
+ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **)
+ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...)
+ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *)
+ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *)
+ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *)
+ldb_msg_add_string: int (struct ldb_message *, const char *, const char *)
+ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **)
+ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *)
+ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *)
+ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *)
+ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *)
+ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **)
+ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *)
+ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *)
+ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *)
+ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double)
+ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int)
+ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t)
+ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *)
+ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int)
+ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t)
+ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *)
+ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *)
+ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *)
+ldb_msg_new: struct ldb_message *(TALLOC_CTX *)
+ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **)
+ldb_msg_remove_attr: void (struct ldb_message *, const char *)
+ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *)
+ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *)
+ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *)
+ldb_msg_sort_elements: void (struct ldb_message *)
+ldb_next_del_trans: int (struct ldb_module *)
+ldb_next_end_trans: int (struct ldb_module *)
+ldb_next_init: int (struct ldb_module *)
+ldb_next_prepare_commit: int (struct ldb_module *)
+ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_request: int (struct ldb_module *, struct ldb_request *)
+ldb_next_start_trans: int (struct ldb_module *)
+ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_options_find: const char *(struct ldb_context *, const char **, const char *)
+ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *)
+ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *)
+ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **)
+ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *)
+ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *)
+ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *)
+ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *)
+ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t)
+ldb_register_backend: int (const char *, ldb_connect_fn, bool)
+ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *)
+ldb_register_hook: int (ldb_hook_fn)
+ldb_register_module: int (const struct ldb_module_ops *)
+ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *)
+ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *)
+ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *)
+ldb_req_get_custom_flags: uint32_t (struct ldb_request *)
+ldb_req_is_untrusted: bool (struct ldb_request *)
+ldb_req_location: const char *(struct ldb_request *)
+ldb_req_mark_trusted: void (struct ldb_request *)
+ldb_req_mark_untrusted: void (struct ldb_request *)
+ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t)
+ldb_req_set_location: void (struct ldb_request *, const char *)
+ldb_request: int (struct ldb_context *, struct ldb_request *)
+ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_done: int (struct ldb_request *, int)
+ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *)
+ldb_request_get_status: int (struct ldb_request *)
+ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *)
+ldb_request_set_state: void (struct ldb_request *, int)
+ldb_reset_err_string: void (struct ldb_context *)
+ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***)
+ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *)
+ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *)
+ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *)
+ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *)
+ldb_schema_attribute_remove: void (struct ldb_context *, const char *)
+ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int)
+ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *)
+ldb_schema_set_override_indexlist: void (struct ldb_context *, bool)
+ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...)
+ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *)
+ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *)
+ldb_set_create_perms: void (struct ldb_context *, unsigned int)
+ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *)
+ldb_set_debug_stderr: int (struct ldb_context *)
+ldb_set_default_dns: void (struct ldb_context *)
+ldb_set_errstring: void (struct ldb_context *, const char *)
+ldb_set_event_context: void (struct ldb_context *, struct tevent_context *)
+ldb_set_flags: void (struct ldb_context *, unsigned int)
+ldb_set_modules_dir: void (struct ldb_context *, const char *)
+ldb_set_opaque: int (struct ldb_context *, const char *, void *)
+ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int)
+ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *)
+ldb_set_utf8_default: void (struct ldb_context *)
+ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t))
+ldb_setup_wellknown_attributes: int (struct ldb_context *)
+ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *)
+ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *)
+ldb_strerror: const char *(int)
+ldb_string_to_time: time_t (const char *)
+ldb_string_utc_to_time: time_t (const char *)
+ldb_timestring: char *(TALLOC_CTX *, time_t)
+ldb_timestring_utc: char *(TALLOC_CTX *, time_t)
+ldb_transaction_cancel: int (struct ldb_context *)
+ldb_transaction_cancel_noerr: int (struct ldb_context *)
+ldb_transaction_commit: int (struct ldb_context *)
+ldb_transaction_prepare_commit: int (struct ldb_context *)
+ldb_transaction_start: int (struct ldb_context *)
+ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
+ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
+ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
+ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
+ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
+ldb_val_string_cmp: int (const struct ldb_val *, const char *)
+ldb_val_to_time: int (const struct ldb_val *, time_t *)
+ldb_valid_attr_name: int (const char *)
+ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list)
+ldb_wait: int (struct ldb_handle *, enum ldb_wait_type)
diff --git a/lib/ldb/ABI/pyldb-util-1.1.30.sigs b/lib/ldb/ABI/pyldb-util-1.1.30.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util-1.1.30.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs b/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs
new file mode 100644
index 00000000000..74d6719d2bc
--- /dev/null
+++ b/lib/ldb/ABI/pyldb-util.py3-1.1.30.sigs
@@ -0,0 +1,2 @@
+pyldb_Dn_FromDn: PyObject *(struct ldb_dn *)
+pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **)
diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index c877f3bc7f6..479e46d38f5 100644
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 APPNAME = 'ldb'
-VERSION = '1.1.29'
+VERSION = '1.1.30'
 
 blddir = 'bin'
 
-- 
2.11.0


From d9adfd7815def25a20054016a7c1c550ba3a4990 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Thu, 30 Mar 2017 13:25:35 +1300
Subject: [PATCH 17/19] schema: Use ldb_schema_set_override_indexlist for
 faster index selection

This allows Samba to provide a binary tree lookup for the existance of an index on the attribute
rather than the O(n) lookup that was being done for each attribute during a search or modify

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 source4/dsdb/schema/schema_init.c | 4 ++++
 source4/dsdb/schema/schema_set.c  | 1 +
 2 files changed, 5 insertions(+)

diff --git a/source4/dsdb/schema/schema_init.c b/source4/dsdb/schema/schema_init.c
index 256c4677452..c76b57cb4a9 100644
--- a/source4/dsdb/schema/schema_init.c
+++ b/source4/dsdb/schema/schema_init.c
@@ -514,6 +514,10 @@ static int dsdb_schema_setup_ldb_schema_attribute(struct ldb_context *ldb,
 		a->flags |= LDB_ATTR_FLAG_SINGLE_VALUE;
 	}
 	
+	if (attr->searchFlags & SEARCH_FLAG_ATTINDEX) {
+		a->flags |= LDB_ATTR_FLAG_INDEXED;
+	}
+
 	
 	return LDB_SUCCESS;
 }
diff --git a/source4/dsdb/schema/schema_set.c b/source4/dsdb/schema/schema_set.c
index 2e688d07546..f43ec5f9c20 100644
--- a/source4/dsdb/schema/schema_set.c
+++ b/source4/dsdb/schema/schema_set.c
@@ -69,6 +69,7 @@ static int dsdb_schema_set_indices_and_attributes(struct ldb_context *ldb, struc
 
 	/* setup our own attribute name to schema handler */
 	ldb_schema_attribute_set_override_handler(ldb, dsdb_attribute_handler_override, schema);
+	ldb_schema_set_override_indexlist(ldb, true);
 
 	if (!write_indices_and_attributes) {
 		return ret;
-- 
2.11.0


From 20be7f77b889dcdddaeb4435a2f63ba489bb7588 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Tue, 28 Mar 2017 21:04:23 +1300
Subject: [PATCH 18/19] process_standard: clean up messaging for children after
 exit()

This makes sure we remove any messaging sockets if a child dies or calls exit()
without running the talloc destructor for messaging

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 source4/smbd/process_standard.c | 3 +++
 source4/smbd/wscript_build      | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/source4/smbd/process_standard.c b/source4/smbd/process_standard.c
index ca93f93ce55..bccddecd928 100644
--- a/source4/smbd/process_standard.c
+++ b/source4/smbd/process_standard.c
@@ -29,6 +29,7 @@
 #include "param/param.h"
 #include "ldb_wrap.h"
 #include "lib/messaging/messaging.h"
+#include "source3/lib/messages_dgm.h"
 
 struct standard_child_state {
 	const char *name;
@@ -85,6 +86,8 @@ static void standard_child_pipe_handler(struct tevent_context *ev,
 	int status = 0;
 	pid_t pid;
 
+	messaging_dgm_cleanup(state->pid);
+
 	/* the child has closed the pipe, assume its dead */
 	errno = 0;
 	pid = waitpid(state->pid, &status, 0);
diff --git a/source4/smbd/wscript_build b/source4/smbd/wscript_build
index ca2039694a4..c28bc1df38a 100644
--- a/source4/smbd/wscript_build
+++ b/source4/smbd/wscript_build
@@ -40,7 +40,7 @@ bld.SAMBA_MODULE('process_model_standard',
                  source='process_standard.c',
                  subsystem='process_model',
                  init_function='process_model_standard_init',
-                 deps='MESSAGING events ldbsamba process_model samba-sockets cluster',
+                 deps='MESSAGING events ldbsamba process_model samba-sockets cluster messages_dgm',
                  internal_module=False
                  )
 
-- 
2.11.0


From 49a7693fb97e1646cd46d0e9221a6fd8a6b29b14 Mon Sep 17 00:00:00 2001
From: Andrew Bartlett <abartlet@xxxxxxxxx>
Date: Tue, 28 Mar 2017 21:55:47 +1300
Subject: [PATCH 19/19] s4-messaging: Add helpful comments

Signed-off-by: Andrew Bartlett <abartlet@xxxxxxxxx>
---
 source4/lib/messaging/messaging.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/source4/lib/messaging/messaging.c b/source4/lib/messaging/messaging.c
index 844466d41c0..72864563e06 100644
--- a/source4/lib/messaging/messaging.c
+++ b/source4/lib/messaging/messaging.c
@@ -388,6 +388,10 @@ struct imessaging_context *imessaging_init(TALLOC_CTX *mem_ctx,
 
 	tdb_flags |= lpcfg_tdb_flags(lp_ctx, 0);
 
+	/*
+	 * This context holds a destructor that cleans up any names
+	 * registered on this context on talloc_free()
+	 */
 	msg->names = server_id_db_init(msg, server_id, lock_dir, 0, tdb_flags);
 	if (msg->names == NULL) {
 		goto fail;
@@ -747,6 +751,9 @@ static int irpc_destructor(struct irpc_request *irpc)
 
 /*
   add a string name that this irpc server can be called on
+
+  It will be removed from the DB either via irpc_remove_name or on
+  talloc_free(msg_ctx->names).
 */
 NTSTATUS irpc_add_name(struct imessaging_context *msg_ctx, const char *name)
 {
-- 
2.11.0

-- 
To unsubscribe from this list go to the following URL and read the
instructions:  https://lists.samba.org/mailman/options/samba