*Use the new Script response from server app api to track the wallet index

* more detailed restore flow (wip)

BTCPayApp.Core/BTCPayServer/ExceptionWrappedHubProxy.cs

@@ -84,7 +84,7 @@ public class ExceptionWrappedHubProxy : IBTCPayAppHubServer
         return await Wrap(async ()=>  await _hubProxy.FetchTxsAndTheirBlockHeads(identifier, txIds, outpoints));
     }
 
-    public async Task<string> DeriveScript(string identifier)
+    public async Task<ScriptResponse> DeriveScript(string identifier)
     {
         return await Wrap(async ()=>  await _hubProxy.DeriveScript(identifier));
     }

BTCPayApp.Core/Data/WalletDerivation.cs

@@ -9,4 +9,6 @@ public class WalletDerivation
     public const string NativeSegwit = "segwit";
     public const string LightningScripts = "lightningScripts";
     // public const string SpendableOutputs = "spendableOutputs";
+    //this is useful when restoring, to tell NBX to generate addresses up to this to prevent address reuse.
+    public int? LastKnownIndex{ get; set; }
 }

BTCPayApp.Core/LDK/LDKFilter.cs

@@ -1,5 +1,4 @@
 using BTCPayApp.Core.Contracts;
-using BTCPayApp.Core.Helpers;
 using org.ldk.structs;
 using Script = NBitcoin.Script;
 
@@ -32,9 +31,15 @@ public class LDKFilter : FilterInterface
 
     public async Task<List<LDKWatchedOutput>> GetWatchedOutputs()
     {
-        return await _configProvider.Get<List<LDKWatchedOutput>?>("ln:watchedOutputs")?? [];
+        return await GetWatchedOutputs(_configProvider);
     }
 
+    public static async Task<List<LDKWatchedOutput>> GetWatchedOutputs(ConfigProvider configProvider)
+    {
+        return await configProvider.Get<List<LDKWatchedOutput>?>("ln:watchedOutputs") ?? [];
+    }
+    
+
     private readonly SemaphoreSlim _semaphore = new(1, 1);
 
     private async Task AddOrUpdateWatchedOutput(LDKWatchedOutput output)

BTCPayApp.Core/LDK/LDKWatchedOutput.cs

@@ -15,7 +15,7 @@ public class LDKWatchedOutput
     public uint256? BlockHash { get; set; }
 
     [JsonConverter(typeof(BitcoinSerializableJsonConverterFactory))]
-    public OutPoint Outpoint { get; set; }
+    public OutPoint? Outpoint { get; set; }
 
     public LDKWatchedOutput()
     {
@@ -28,5 +28,9 @@ public class LDKWatchedOutput
             ? new uint256(some.some)
             : null;
         Outpoint = watchedOutput.get_outpoint().Outpoint();
+    } 
+    public LDKWatchedOutput(Script script)
+    {
+        Script = script;
     }
 }

BTCPayApp.Core/Wallet/OnChainWalletManager.cs

@@ -3,6 +3,7 @@ using BTCPayApp.Core.BTCPayServer;
 using BTCPayApp.Core.Contracts;
 using BTCPayApp.Core.Data;
 using BTCPayApp.Core.Helpers;
+using BTCPayApp.Core.LDK;
 using BTCPayServer.Client.App;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
@@ -106,7 +107,7 @@ public class OnChainWalletManager : BaseHostedService
     private async Task OnStateChanged(object? sender, (OnChainWalletState Old, OnChainWalletState New) e)
     {
         var config = await GetConfig();
-        if (e is {New: OnChainWalletState.Loaded} && IsConfigured(config))
+        if (e is { New: OnChainWalletState.Loaded } && IsConfigured(config))
         {
             await Track();
         }
@@ -119,45 +120,69 @@ public class OnChainWalletManager : BaseHostedService
 
     public async Task Restore()
     {
-        throw new NotImplementedException("we're not there yet");
-
-        var config = await GetConfig();
-        if (config is null || !IsConfigured(config))
-        {
-            throw new InvalidOperationException("Cannot restore wallet in current state");
-        }
-
-        // step1: Track our derivations
-        //for groups, we need to generate a new one and replace the local one with its identifier
-        //for derivations, we need to call track with the latest index
-        //for groups, we need to fetch all tracked scripts and add them to the group tracked source
-        // step2: import the UTXOS
-        // step3: sync the backup data
         
-        
-        
-        /*
-        await _controlSemaphore.WaitAsync();
         try
         {
-
+            if (_state != OnChainWalletState.WaitingForConnection)
+            {
+                throw new InvalidOperationException("Cannot restore wallet in current state");
+            }
+            await _controlSemaphore.WaitAsync();
 
             var config = await GetConfig();
-            var missingIds = await Track();
-            foreach (var missing in missingIds)
+            if (config is null || !IsConfigured(config))
             {
-                var wd = config.Derivations.First(pair => pair.Value.Identifier == missing).Value;
-                if (wd.Descriptor is null)
-                {
-                    // track and take the new identifier
-
-                }
-
-                //import utxos for the missing id
-                //ask ldk for the scipts we should be tacking and add them all
+                throw new InvalidOperationException("Cannot restore wallet in current state");
             }
 
-            //if it is a wallet without a derivation, we generate a new goup for it
+            // step1: Track our derivations
+            //for groups, we need to generate a new one and replace the local one with its identifier
+            //for derivations, we need to call track with the latest index
+            //for groups, we need to fetch all tracked scripts and add them to the group tracked source
+            // step2: import the UTXOS
+            // step3: sync the backup data
+
+            var identifiers = config.Derivations.Select(pair => pair.Value.Identifier).ToArray();
+            var response = await HubProxy.Handshake(new AppHandshake
+            {
+                Identifiers = identifiers
+            });
+
+            var missing = config.Derivations
+                .Where(pair => response.IdentifiersAcknowledged?.Contains(pair.Value.Identifier) is not true)
+                .ToList();
+
+            foreach (var x in missing)
+            {
+                var result = await HubProxy.Pair(new PairRequest
+                {
+                    Derivations = new Dictionary<string, DerivationItem?>()
+                    {
+                        [x.Key] = new()
+                        {
+                            Descriptor = x.Value.Descriptor,
+                            Index = x.Value.LastKnownIndex ?? 0
+                        }
+                    }
+                });
+
+                config.Derivations[x.Key] = new WalletDerivation
+                {
+                    Name = x.Value.Name,
+                    Descriptor = x.Value.Descriptor,
+                    Identifier = result[x.Key]
+                };
+
+                if (x.Key == WalletDerivation.LightningScripts)
+                {
+                    var scripts = await LDKFilter.GetWatchedOutputs(_configProvider);
+                    await HubProxy.TrackScripts(config.Derivations[x.Key].Identifier,
+                        scripts.Select(output => output.Script.ToHex()).ToArray());
+                }
+            }
+
+
+            await _configProvider.Set(WalletConfig.Key, config, true);
         }
         finally
         {
@@ -172,7 +197,8 @@ public class OnChainWalletManager : BaseHostedService
         await _controlSemaphore.WaitAsync();
         try
         {
-            if (State != OnChainWalletState.NotConfigured || ReportedNetwork == null || HubProxy == null || !IsHubConnected ||
+            if (State != OnChainWalletState.NotConfigured || ReportedNetwork == null || HubProxy == null ||
+                !IsHubConnected ||
                 IsConfigured(await GetConfig()) || await GetBestBlock() is not { } block)
                 throw new InvalidOperationException("Cannot generate wallet in current state");
 
@@ -186,7 +212,7 @@ public class OnChainWalletManager : BaseHostedService
             var snapshot = new BlockSnapshot()
             {
                 BlockHash = uint256.Parse(block.BlockHash),
-                BlockHeight = (uint) block.BlockHeight
+                BlockHeight = (uint)block.BlockHeight
             };
             var walletConfig = new WalletConfig
             {
@@ -211,7 +237,10 @@ public class OnChainWalletManager : BaseHostedService
 
             var result = await HubProxy.Pair(new PairRequest
             {
-                Derivations = walletConfig.Derivations.ToDictionary(pair => pair.Key, pair => pair.Value.Descriptor)
+                Derivations = walletConfig.Derivations.ToDictionary(pair => pair.Key, pair => new DerivationItem()
+                {
+                    Descriptor = pair.Value.Descriptor
+                })
             });
             foreach (var keyValuePair in result)
             {
@@ -239,7 +268,8 @@ public class OnChainWalletManager : BaseHostedService
         try
         {
             var config = await GetConfig();
-            if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null || !IsConfigured(config))
+            if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null ||
+                !IsConfigured(config))
                 throw new InvalidOperationException("Cannot add deriv in current state");
 
             if (config.Derivations.ContainsKey(key))
@@ -247,9 +277,12 @@ public class OnChainWalletManager : BaseHostedService
 
             var result = await HubProxy.Pair(new PairRequest
             {
-                Derivations = new Dictionary<string, string?>
+                Derivations = new Dictionary<string, DerivationItem?>()
                 {
-                    [key] = descriptor
+                    [key] = new DerivationItem()
+                    {
+                        Descriptor = descriptor
+                    }
                 }
             });
 
@@ -307,7 +340,7 @@ public class OnChainWalletManager : BaseHostedService
             return [];
 
         var config = await GetConfig();
-        if (config is null ||!IsConfigured(config))
+        if (config is null || !IsConfigured(config))
             return [];
 
         var identifiers = config.Derivations.Select(pair => pair.Value.Identifier).ToArray();
@@ -367,14 +400,14 @@ public class OnChainWalletManager : BaseHostedService
                 BlockSnapshot = new BlockSnapshot
                 {
                     BlockHash = uint256.Parse(bb.BlockHash),
-                    BlockHeight = (uint) bb.BlockHeight
+                    BlockHeight = (uint)bb.BlockHeight
                 },
                 Coins = utxos.ToDictionary(kv => kv.Key,
                     kv => kv.Value.Select(coin => new SavedCoin()
-                {
-                    Outpoint = OutPoint.Parse(coin.Outpoint),
-                    Path = coin.Path is null ? null : KeyPath.Parse(coin.Path)
-                }).ToArray())
+                    {
+                        Outpoint = OutPoint.Parse(coin.Outpoint),
+                        Path = coin.Path is null ? null : KeyPath.Parse(coin.Path)
+                    }).ToArray())
             };
             await _configProvider.Set(WalletConfig.Key, config, true);
             OnSnapshotUpdate?.Invoke(this, config.CoinSnapshot);
@@ -388,13 +421,25 @@ public class OnChainWalletManager : BaseHostedService
 
     public async Task<BitcoinAddress?> DeriveScript(string derivation)
     {
-        var config = await GetConfig();
-        if (State != OnChainWalletState.Loaded || ReportedNetwork == null || HubProxy == null || !IsHubConnected || config is null || !IsConfigured(config))
-            throw new InvalidOperationException("Cannot derive script in current state");
+        try
+        {
+            await _controlSemaphore.WaitAsync();
+            var config = await GetConfig();
+            if (State != OnChainWalletState.Loaded || ReportedNetwork == null || HubProxy == null || !IsHubConnected ||
+                config is null || !IsConfigured(config))
+                throw new InvalidOperationException("Cannot derive script in current state");
 
-        var identifier = config.Derivations[derivation].Identifier;
-        var addr = await HubProxy.DeriveScript(identifier);
-        return Script.FromHex(addr).GetDestinationAddress(ReportedNetwork);
+            var identifier = config.Derivations[derivation].Identifier;
+            var addr = await HubProxy.DeriveScript(identifier);
+            var keyPath = KeyPath.Parse(addr.KeyPath);
+            config.Derivations[derivation].LastKnownIndex = (int?)keyPath.Indexes.Last();
+            await _configProvider.Set(WalletConfig.Key, config, true);
+            return Script.FromHex(addr.Script).GetDestinationAddress(ReportedNetwork);
+        }
+        finally
+        {
+            _controlSemaphore.Release();
+        }
     }
 
     public async Task<PSBT?> SignTransaction(byte[] psbtBytes)
@@ -409,7 +454,8 @@ public class OnChainWalletManager : BaseHostedService
     public async Task<PSBT?> SignTransaction(PSBT psbt)
     {
         var config = await GetConfig();
-        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null || !IsConfigured(config))
+        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null ||
+            !IsConfigured(config))
             throw new InvalidOperationException("Cannot sign transaction in current state");
 
         var identifiers = config.Derivations.Select(derivation => derivation.Value.Identifier).ToArray();
@@ -492,7 +538,8 @@ public class OnChainWalletManager : BaseHostedService
     public async Task<Dictionary<string, TxResp[]>?> GetTransactions()
     {
         var config = await GetConfig();
-        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null || !IsConfigured(config))
+        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null ||
+            !IsConfigured(config))
             throw new InvalidOperationException("Cannot get transactions in current state");
 
         var identifiersWhichWeCanDeriveKeysFor = config.Derivations.Values
@@ -509,22 +556,26 @@ public class OnChainWalletManager : BaseHostedService
     public async Task<IEnumerable<ICoin>> GetUTXOS()
     {
         var config = await GetConfig();
-        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null || !IsConfigured(config))
+        if (State != OnChainWalletState.Loaded || HubProxy == null || !IsHubConnected || config is null ||
+            !IsConfigured(config))
             throw new InvalidOperationException("Cannot get UTXOS in current state");
 
         var identifiers = config.Derivations.Values.Select(derivation => derivation.Identifier).ToArray();
         var utxos = await HubProxy.GetUTXOs(identifiers);
         var identifiersWhichWeCanDeriveKeysFor = config.Derivations.Values
-            .Where(derivation => derivation.Descriptor is not null).Select(derivation => derivation.Identifier.ToLowerInvariant())
+            .Where(derivation => derivation.Descriptor is not null)
+            .Select(derivation => derivation.Identifier.ToLowerInvariant())
             .ToArray();
         var result = new List<ICoin>();
 
         var utxosThatWeCanDeriveKeysFor =
-            utxos.Where(utxo => identifiersWhichWeCanDeriveKeysFor.Contains(utxo.Key.ToLowerInvariant())).ToDictionary(pair => pair.Key, pair => pair.Value);
+            utxos.Where(utxo => identifiersWhichWeCanDeriveKeysFor.Contains(utxo.Key.ToLowerInvariant()))
+                .ToDictionary(pair => pair.Key, pair => pair.Value);
         foreach (var kp in utxosThatWeCanDeriveKeysFor)
         {
             var derivation =
-                config.Derivations.Values.First(derivation => derivation.Identifier.Equals(kp.Key, StringComparison.InvariantCultureIgnoreCase));
+                config.Derivations.Values.First(derivation =>
+                    derivation.Identifier.Equals(kp.Key, StringComparison.InvariantCultureIgnoreCase));
             var data = derivation.Descriptor.ExtractFromDescriptor(config.NBitcoinNetwork);
             if (data is null)
                 continue;

submodules/btcpayserver

@@ -1 +1 @@
-Subproject commit 853a6ec9dc79e6c49c8f88e3e5859f5bf84abc8f
+Subproject commit d949117b029967fa332fd7a5141c3a3b9f094533