moved akka conf to eclair-node application.conf

use a local queue for outgoing rpc requests. this should be a better solution than
inceasing the number of concurrent requests.
see https://doc.akka.io/docs/akka-http/current/scala/http/client-side/host-level.html#using-the-host-level-api-with-a-queue
for more information.

README.md

@@ -27,7 +27,7 @@ Please see the latest [release note](https://github.com/ACINQ/eclair/releases) f
 
 ## Installation
 
-:warning: **Those are valid for the most up-to-date, unreleased, version of eclair. Here are the [instructions for Eclair 0.2-alpha4](https://github.com/ACINQ/eclair/blob/v0.2-alpha4/README.md#installation)**.
+:warning: **Those are valid for the most up-to-date, unreleased, version of eclair. Here are the [instructions for Eclair 0.2-alpha5](https://github.com/ACINQ/eclair/blob/v0.2-alpha5/README.md#installation)**.
 
 ### Configuring Bitcoin Core
 
@@ -92,6 +92,8 @@ name                         | description               | default value
  eclair.bitcoind.rpcpassword | Bitcoin Core RPC password | bar
  eclair.bitcoind.zmq         | Bitcoin Core ZMQ address  | tcp://127.0.0.1:29000
 
+Quotes are not required unless the value contains special characters. Full syntax guide [here](https://github.com/lightbend/config/blob/master/HOCON.md).
+
 → see [`reference.conf`](eclair-core/src/main/resources/reference.conf) for full reference. There are many more options!
 
 #### Java Environment Variables
@@ -116,8 +118,8 @@ java -Declair.datadir=/tmp/node1 -jar eclair-node-gui-<version>-<commit_id>.jar
  method       |  params                                       | description
  -------------|-----------------------------------------------|-----------------------------------------------------------
   getinfo     |                                               | return basic node information (id, chain hash, current block height) 
-  connect     | host, port, nodeId                            | connect to another lightning node through a secure connection
-  open        | host, port, nodeId, fundingSatoshis, pushMsat | opens a channel with another lightning node
+  connect     | nodeId, host, port                            | connect to another lightning node through a secure connection
+  open        | nodeId, host, port, fundingSatoshis, pushMsat | opens a channel with another lightning node
   peers       |                                               | list existing local peers
   channels    |                                               | list existing local channels
   channel     | channelId                                     | retrieve detailed information about a given channel

eclair-core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>fr.acinq.eclair</groupId>
         <artifactId>eclair_2.11</artifactId>
-        <version>0.2-alpha5</version>
+        <version>0.2-SNAPSHOT</version>
     </parent>
 
     <artifactId>eclair-core_2.11</artifactId>
@@ -83,9 +83,9 @@
         <profile>
             <id>Mac</id>
             <activation>
-              <os>
-                <family>mac</family>
-              </os>
+                <os>
+                    <family>mac</family>
+                </os>
             </activation>
             <properties>
                 <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.14.0/bitcoin-0.14.0-osx64.tar.gz

eclair-core/src/main/resources/electrum/servers_regtest.json

@@ -0,0 +1,10 @@
+{
+  "127.0.0.1": {
+    "t": "51001",
+    "s": "51002"
+  },
+  "10.0.2.2": {
+    "t": "51001",
+    "s": "51002"
+  }
+}

eclair-core/src/main/resources/electrum/servers_testnet.json

@@ -0,0 +1,14 @@
+{
+  "testnetnode.arihanc.com": {
+    "t": "51001",
+    "s": "51002"
+  },
+  "testnet.hsmiths.com": {
+    "t": "53011",
+    "s": "53012"
+  },
+  "electrum.akinbo.org": {
+    "t": "51001",
+    "s": "51002"
+  }
+}

eclair-core/src/main/resources/reference.conf

@@ -1,17 +1,20 @@
 eclair {
 
-  chain = "test"
-  spv = false // experimental!! do not use
+  chain = "test" // "regtest" for regtest, "test" for testnet. Livenet is not supported.
 
   server {
     public-ips = [] // external ips, will be announced on the network
     binding-ip = "0.0.0.0"
     port = 9735
   }
+
   api {
     binding-ip = "127.0.0.1"
     port = 8080
   }
+
+  watcher-type = "bitcoind" // other *experimental* values include "bitcoinj" or "electrum"
+
   bitcoind {
     host = "localhost"
     rpcport = 18332
@@ -29,6 +32,17 @@ eclair {
     ]
   }
 
+  default-feerates { // those are in satoshis per byte
+    delay-blocks {
+      1 = 210
+      2 = 180
+      6 = 150
+      12 = 110
+      36 = 50
+      72 = 20
+    }
+  }
+
   node-alias = "eclair"
   node-color = "49daaa"
   global-features = ""
@@ -67,21 +81,4 @@ eclair {
   auto-reconnect = true
 
   payment-handler = "local"
-}
-akka {
-  loggers = ["akka.event.slf4j.Slf4jLogger"]
-  loglevel = "DEBUG"
-
-  actor {
-    debug {
-      # enable DEBUG logging of all LoggingFSMs for events, transitions and timers
-      fsm = on
-    }
-  }
-
-  http {
-    host-connection-pool {
-      max-open-requests = 64
-    }
-  }
 }

eclair-core/src/main/scala/fr/acinq/eclair/FSMDiagnosticActorLogging.scala

@@ -8,8 +8,11 @@ import akka.event.{DiagnosticLoggingAdapter, LoggingAdapter}
   * See https://groups.google.com/forum/#!topic/akka-user/0CxR8CImr4Q
   */
 trait FSMDiagnosticActorLogging[S, D] extends FSM[S, D] {
+
   import akka.event.Logging._
+
   val diagLog: DiagnosticLoggingAdapter = akka.event.Logging(this)
+
   def mdc(currentMessage: Any): MDC = emptyMDC
 
   override def log: LoggingAdapter = diagLog

eclair-core/src/main/scala/fr/acinq/eclair/Features.scala

@@ -2,7 +2,6 @@ package fr.acinq.eclair
 
 
 import java.util.BitSet
-import java.util.function.IntPredicate
 
 import fr.acinq.bitcoin.BinaryData
 
@@ -20,14 +19,14 @@ object Features {
     * @param features feature bits
     * @return true if an initial dump of the routing table is requested
     */
-  def initialRoutingSync(features: BitSet) : Boolean = features.get(INITIAL_ROUTING_SYNC_BIT_OPTIONAL)
+  def initialRoutingSync(features: BitSet): Boolean = features.get(INITIAL_ROUTING_SYNC_BIT_OPTIONAL)
 
   /**
     *
     * @param features feature bits
     * @return true if an initial dump of the routing table is requested
     */
-  def initialRoutingSync(features: BinaryData) : Boolean = initialRoutingSync(BitSet.valueOf(features.reverse.toArray))
+  def initialRoutingSync(features: BinaryData): Boolean = initialRoutingSync(BitSet.valueOf(features.reverse.toArray))
 
   /**
     * Check that the features that we understand are correctly specified, and that there are no mandatory features that
@@ -35,7 +34,7 @@ object Features {
     */
   def areSupported(bitset: BitSet): Boolean = {
     // for now there is no mandatory feature bit, so we don't support features with any even bit set
-    for(i <- 0 until bitset.length() by 2) {
+    for (i <- 0 until bitset.length() by 2) {
       if (bitset.get(i)) return false
     }
     return true

eclair-core/src/main/scala/fr/acinq/eclair/Globals.scala

@@ -1,7 +1,8 @@
 package fr.acinq.eclair
 
-import java.util.concurrent.atomic.AtomicLong
+import java.util.concurrent.atomic.{AtomicLong, AtomicReference}
 
+import fr.acinq.eclair.blockchain.fee.{FeeratesPerByte, FeeratesPerKw}
 
 /**
   * Created by PM on 25/01/2016.
@@ -11,16 +12,21 @@ object Globals {
   /**
     * This counter holds the current blockchain height.
     * It is mainly used to calculate htlc expiries.
-    * The value is updated by the [[fr.acinq.eclair.blockchain.ZmqWatcher]] and read by all actors, hence it needs to be thread-safe.
+    * The value is read by all actors, hence it needs to be thread-safe.
     */
   val blockCount = new AtomicLong(0)
 
   /**
-    * This counter holds the current feeratePerKw.
-    * It is used to maintain an up-to-date fee in commitment tx so that they get confirmed fast enough.
-    * The value is updated by the [[fr.acinq.eclair.blockchain.ZmqWatcher]] and read by all actors, hence it needs to be thread-safe.
+    * This holds the current feerates, in satoshi-per-bytes.
+    * The value is read by all actors, hence it needs to be thread-safe.
     */
-  val feeratePerKw = new AtomicLong(0)
+  val feeratesPerByte = new AtomicReference[FeeratesPerByte](null)
+
+  /**
+    * This holds the current feerates, in satoshi-per-kw.
+    * The value is read by all actors, hence it needs to be thread-safe.
+    */
+  val feeratesPerKw = new AtomicReference[FeeratesPerKw](null)
 }
 
 

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -10,8 +10,9 @@ import com.typesafe.config.{Config, ConfigFactory}
 import fr.acinq.bitcoin.Crypto.PrivateKey
 import fr.acinq.bitcoin.DeterministicWallet.ExtendedPrivateKey
 import fr.acinq.bitcoin.{BinaryData, Block, DeterministicWallet}
+import fr.acinq.eclair.NodeParams.WatcherType
 import fr.acinq.eclair.db._
-import fr.acinq.eclair.db.sqlite.{SqliteChannelsDb, SqliteNetworkDb, SqlitePeersDb}
+import fr.acinq.eclair.db.sqlite.{SqliteChannelsDb, SqliteNetworkDb, SqlitePeersDb, SqlitePreimagesDb}
 
 import scala.collection.JavaConversions._
 import scala.concurrent.duration.FiniteDuration
@@ -41,6 +42,7 @@ case class NodeParams(extendedPrivateKey: ExtendedPrivateKey,
                       channelsDb: ChannelsDb,
                       peersDb: PeersDb,
                       networkDb: NetworkDb,
+                      preimagesDb: PreimagesDb,
                       routerBroadcastInterval: FiniteDuration,
                       routerValidateInterval: FiniteDuration,
                       pingInterval: FiniteDuration,
@@ -50,10 +52,18 @@ case class NodeParams(extendedPrivateKey: ExtendedPrivateKey,
                       chainHash: BinaryData,
                       channelFlags: Byte,
                       channelExcludeDuration: FiniteDuration,
-                      spv: Boolean)
+                      watcherType: WatcherType)
 
 object NodeParams {
 
+  sealed trait WatcherType
+
+  object BITCOIND extends WatcherType
+
+  object BITCOINJ extends WatcherType
+
+  object ELECTRUM extends WatcherType
+
   /**
     * Order of precedence for the configuration parameters:
     * 1) Java environment variables (-D...)
@@ -93,10 +103,17 @@ object NodeParams {
     val channelsDb = new SqliteChannelsDb(sqlite)
     val peersDb = new SqlitePeersDb(sqlite)
     val networkDb = new SqliteNetworkDb(sqlite)
+    val preimagesDb = new SqlitePreimagesDb(sqlite)
 
     val color = BinaryData(config.getString("node-color"))
     require(color.size == 3, "color should be a 3-bytes hex buffer")
 
+    val watcherType = config.getString("watcher-type") match {
+      case "bitcoinj" => BITCOINJ
+      case "electrum" => ELECTRUM
+      case _ => BITCOIND
+    }
+
     NodeParams(
       extendedPrivateKey = extendedPrivateKey,
       privateKey = extendedPrivateKey.privateKey,
@@ -120,6 +137,7 @@ object NodeParams {
       channelsDb = channelsDb,
       peersDb = peersDb,
       networkDb = networkDb,
+      preimagesDb = preimagesDb,
       routerBroadcastInterval = FiniteDuration(config.getDuration("router-broadcast-interval").getSeconds, TimeUnit.SECONDS),
       routerValidateInterval = FiniteDuration(config.getDuration("router-validate-interval").getSeconds, TimeUnit.SECONDS),
       pingInterval = FiniteDuration(config.getDuration("ping-interval").getSeconds, TimeUnit.SECONDS),
@@ -129,6 +147,6 @@ object NodeParams {
       chainHash = chainHash,
       channelFlags = config.getInt("channel-flags").toByte,
       channelExcludeDuration = FiniteDuration(config.getDuration("channel-exclude-duration").getSeconds, TimeUnit.SECONDS),
-      spv = config.getBoolean("spv"))
+      watcherType = watcherType)
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -9,14 +9,16 @@ import akka.pattern.after
 import akka.stream.{ActorMaterializer, BindFailedException}
 import akka.util.Timeout
 import com.typesafe.config.{Config, ConfigFactory}
-import fr.acinq.bitcoin.BinaryData
+import fr.acinq.bitcoin.{BinaryData, Block}
+import fr.acinq.eclair.NodeParams.{BITCOIND, BITCOINJ, ELECTRUM}
 import fr.acinq.eclair.api.{GetInfoResponse, Service}
-import fr.acinq.eclair.blockchain._
-import fr.acinq.eclair.blockchain.fee.{BitpayInsightFeeProvider, ConstantFeeProvider}
-import fr.acinq.eclair.blockchain.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
-import fr.acinq.eclair.blockchain.spv.BitcoinjKit
-import fr.acinq.eclair.blockchain.wallet.{BitcoinCoreWallet, BitcoinjWallet, EclairWallet}
-import fr.acinq.eclair.blockchain.zmq.ZMQActor
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
+import fr.acinq.eclair.blockchain.bitcoind.zmq.ZMQActor
+import fr.acinq.eclair.blockchain.bitcoind.{BitcoinCoreWallet, ZmqWatcher}
+import fr.acinq.eclair.blockchain.bitcoinj.{BitcoinjKit, BitcoinjWallet, BitcoinjWatcher}
+import fr.acinq.eclair.blockchain.electrum.{ElectrumClient, ElectrumEclairWallet, ElectrumWallet, ElectrumWatcher}
+import fr.acinq.eclair.blockchain.fee.{ConstantFeeProvider, _}
+import fr.acinq.eclair.blockchain.{EclairWallet, _}
 import fr.acinq.eclair.channel.Register
 import fr.acinq.eclair.io.{Server, Switchboard}
 import fr.acinq.eclair.payment._
@@ -37,7 +39,6 @@ class Setup(datadir: File, overrideDefaults: Config = ConfigFactory.empty(), act
 
   val config = NodeParams.loadConfiguration(datadir, overrideDefaults)
   val nodeParams = NodeParams.makeNodeParams(datadir, config)
-  val spv = config.getBoolean("spv")
   val chain = config.getString("chain")
 
   // early checks
@@ -57,64 +58,85 @@ class Setup(datadir: File, overrideDefaults: Config = ConfigFactory.empty(), act
   implicit val formats = org.json4s.DefaultFormats
   implicit val ec = ExecutionContext.Implicits.global
 
-  val bitcoin = if (spv) {
-    logger.warn("EXPERIMENTAL SPV MODE ENABLED!!!")
-    val staticPeers = config.getConfigList("bitcoinj.static-peers").map(c => new InetSocketAddress(c.getString("host"), c.getInt("port"))).toList
-    logger.info(s"using staticPeers=$staticPeers")
-    val bitcoinjKit = new BitcoinjKit(chain, datadir, staticPeers)
-    bitcoinjKit.startAsync()
-    Await.ready(bitcoinjKit.initialized, 10 seconds)
-    Left(bitcoinjKit)
-  } else {
-    val bitcoinClient = new ExtendedBitcoinClient(new BitcoinJsonRPCClient(
-      user = config.getString("bitcoind.rpcuser"),
-      password = config.getString("bitcoind.rpcpassword"),
-      host = config.getString("bitcoind.host"),
-      port = config.getInt("bitcoind.rpcport")))
-    val future = for {
-      json <- bitcoinClient.rpcClient.invoke("getblockchaininfo").recover { case _ => throw BitcoinRPCConnectionException }
-      progress = (json \ "verificationprogress").extract[Double]
-      chainHash <- bitcoinClient.rpcClient.invoke("getblockhash", 0).map(_.extract[String]).map(BinaryData(_)).map(x => BinaryData(x.reverse))
-      bitcoinVersion <- bitcoinClient.rpcClient.invoke("getnetworkinfo").map(json => (json \ "version")).map(_.extract[String])
-    } yield (progress, chainHash, bitcoinVersion)
-    // blocking sanity checks
-    val (progress, chainHash, bitcoinVersion) = Await.result(future, 10 seconds)
-    assert(chainHash == nodeParams.chainHash, s"chainHash mismatch (conf=${nodeParams.chainHash} != bitcoind=$chainHash)")
-    assert(progress > 0.99, "bitcoind should be synchronized")
-    // TODO: add a check on bitcoin version?
-    Right(bitcoinClient)
+  val bitcoin = nodeParams.watcherType match {
+    case BITCOIND =>
+      val bitcoinClient = new ExtendedBitcoinClient(new BitcoinJsonRPCClient(
+        user = config.getString("bitcoind.rpcuser"),
+        password = config.getString("bitcoind.rpcpassword"),
+        host = config.getString("bitcoind.host"),
+        port = config.getInt("bitcoind.rpcport")))
+      val future = for {
+        json <- bitcoinClient.rpcClient.invoke("getblockchaininfo").recover { case _ => throw BitcoinRPCConnectionException }
+        progress = (json \ "verificationprogress").extract[Double]
+        chainHash <- bitcoinClient.rpcClient.invoke("getblockhash", 0).map(_.extract[String]).map(BinaryData(_)).map(x => BinaryData(x.reverse))
+        bitcoinVersion <- bitcoinClient.rpcClient.invoke("getnetworkinfo").map(json => (json \ "version")).map(_.extract[String])
+      } yield (progress, chainHash, bitcoinVersion)
+      // blocking sanity checks
+      val (progress, chainHash, bitcoinVersion) = Await.result(future, 10 seconds)
+      assert(chainHash == nodeParams.chainHash, s"chainHash mismatch (conf=${nodeParams.chainHash} != bitcoind=$chainHash)")
+      assert(progress > 0.99, "bitcoind should be synchronized")
+      // TODO: add a check on bitcoin version?
+      Bitcoind(bitcoinClient)
+    case BITCOINJ =>
+      logger.warn("EXPERIMENTAL BITCOINJ MODE ENABLED!!!")
+      val staticPeers = config.getConfigList("bitcoinj.static-peers").map(c => new InetSocketAddress(c.getString("host"), c.getInt("port"))).toList
+      logger.info(s"using staticPeers=$staticPeers")
+      val bitcoinjKit = new BitcoinjKit(chain, datadir, staticPeers)
+      bitcoinjKit.startAsync()
+      Await.ready(bitcoinjKit.initialized, 10 seconds)
+      Bitcoinj(bitcoinjKit)
+    case ELECTRUM =>
+      logger.warn("EXPERIMENTAL ELECTRUM MODE ENABLED!!!")
+      val addressesFile = chain match {
+        case "test" => "/electrum/servers_testnet.json"
+        case "regtest" => "/electrum/servers_regtest.json"
+      }
+      val stream = classOf[Setup].getResourceAsStream(addressesFile)
+      val addresses = ElectrumClient.readServerAddresses(stream)
+      val electrumClient = system.actorOf(SimpleSupervisor.props(Props(new ElectrumClient(addresses)), "electrum-client", SupervisorStrategy.Resume))
+      Electrum(electrumClient)
   }
 
   def bootstrap: Future[Kit] = {
     val zmqConnected = Promise[Boolean]()
     val tcpBound = Promise[Unit]()
 
-    val defaultFeeratePerKb = config.getLong("default-feerate-per-kb")
-    Globals.feeratePerKw.set(feerateKb2Kw(defaultFeeratePerKb))
-    logger.info(s"initial feeratePerKw=${Globals.feeratePerKw.get()}")
-    val feeProvider = chain match {
-      case "regtest" => new ConstantFeeProvider(defaultFeeratePerKb)
-      case _ => new BitpayInsightFeeProvider()
+    val defaultFeerates = FeeratesPerByte(block_1 = config.getLong("default-feerates.delay-blocks.1"), blocks_2 = config.getLong("default-feerates.delay-blocks.2"), blocks_6 = config.getLong("default-feerates.delay-blocks.6"), blocks_12 = config.getLong("default-feerates.delay-blocks.12"), blocks_36 = config.getLong("default-feerates.delay-blocks.36"), blocks_72 = config.getLong("default-feerates.delay-blocks.72"))
+    Globals.feeratesPerByte.set(defaultFeerates)
+    Globals.feeratesPerKw.set(FeeratesPerKw(defaultFeerates))
+    logger.info(s"initial feeratesPerByte=${Globals.feeratesPerByte.get()}")
+    val feeProvider = (chain, bitcoin) match {
+      case ("regtest", _) => new ConstantFeeProvider(defaultFeerates)
+      case (_, Bitcoind(client)) => new FallbackFeeProvider(new EarnDotComFeeProvider() :: new BitcoinCoreFeeProvider(client.rpcClient, defaultFeerates) :: new ConstantFeeProvider(defaultFeerates) :: Nil) // order matters!
+      case _ => new FallbackFeeProvider(new EarnDotComFeeProvider() :: new ConstantFeeProvider(defaultFeerates) :: Nil) // order matters!
     }
-    system.scheduler.schedule(0 seconds, 10 minutes)(feeProvider.getFeeratePerKB.map {
-      case feeratePerKB =>
-        Globals.feeratePerKw.set(feerateKb2Kw(feeratePerKB))
-        system.eventStream.publish(CurrentFeerate(Globals.feeratePerKw.get()))
-        logger.info(s"current feeratePerKw=${Globals.feeratePerKw.get()}")
+    system.scheduler.schedule(0 seconds, 10 minutes)(feeProvider.getFeerates.map {
+      case feerates: FeeratesPerByte =>
+        Globals.feeratesPerByte.set(feerates)
+        Globals.feeratesPerKw.set(FeeratesPerKw(defaultFeerates))
+        system.eventStream.publish(CurrentFeerates(Globals.feeratesPerKw.get))
+        logger.info(s"current feeratesPerByte=${Globals.feeratesPerByte.get()}")
     })
 
     val watcher = bitcoin match {
-      case Left(bitcoinj) =>
-        zmqConnected.success(true)
-        system.actorOf(SimpleSupervisor.props(SpvWatcher.props(bitcoinj), "watcher", SupervisorStrategy.Resume))
-      case Right(bitcoinClient) =>
+      case Bitcoind(bitcoinClient) =>
         system.actorOf(SimpleSupervisor.props(Props(new ZMQActor(config.getString("bitcoind.zmq"), Some(zmqConnected))), "zmq", SupervisorStrategy.Restart))
         system.actorOf(SimpleSupervisor.props(ZmqWatcher.props(bitcoinClient), "watcher", SupervisorStrategy.Resume))
+      case Bitcoinj(bitcoinj) =>
+        zmqConnected.success(true)
+        system.actorOf(SimpleSupervisor.props(BitcoinjWatcher.props(bitcoinj), "watcher", SupervisorStrategy.Resume))
+      case Electrum(electrumClient) =>
+        zmqConnected.success(true)
+        system.actorOf(SimpleSupervisor.props(Props(new ElectrumWatcher(electrumClient)), "watcher", SupervisorStrategy.Resume))
     }
 
     val wallet = bitcoin match {
-      case Left(bitcoinj) => new BitcoinjWallet(bitcoinj.initialized.map(_ => bitcoinj.wallet()))
-      case Right(bitcoinClient) => new BitcoinCoreWallet(bitcoinClient.rpcClient, watcher)
+      case Bitcoind(bitcoinClient) => new BitcoinCoreWallet(bitcoinClient.rpcClient, watcher)
+      case Bitcoinj(bitcoinj) => new BitcoinjWallet(bitcoinj.initialized.map(_ => bitcoinj.wallet()))
+      case Electrum(electrumClient) =>
+        val electrumSeedPath = new File(datadir, "electrum_seed.dat")
+        val electrumWallet = system.actorOf(ElectrumWallet.props(electrumSeedPath, electrumClient, ElectrumWallet.WalletParameters(Block.RegtestGenesisBlock.hash, allowSpendUnconfirmed = true)), "electrum-wallet")
+        new ElectrumEclairWallet(electrumWallet)
     }
     wallet.getFinalAddress.map {
       case address => logger.info(s"initial wallet address=$address")
@@ -125,7 +147,7 @@ class Setup(datadir: File, overrideDefaults: Config = ConfigFactory.empty(), act
       case "noop" => Props[NoopPaymentHandler]
     }, "payment-handler", SupervisorStrategy.Resume))
     val register = system.actorOf(SimpleSupervisor.props(Props(new Register), "register", SupervisorStrategy.Resume))
-    val relayer = system.actorOf(SimpleSupervisor.props(Relayer.props(nodeParams.privateKey, paymentHandler), "relayer", SupervisorStrategy.Resume))
+    val relayer = system.actorOf(SimpleSupervisor.props(Relayer.props(nodeParams, register, paymentHandler), "relayer", SupervisorStrategy.Resume))
     val router = system.actorOf(SimpleSupervisor.props(Router.props(nodeParams, watcher), "router", SupervisorStrategy.Resume))
     val switchboard = system.actorOf(SimpleSupervisor.props(Switchboard.props(nodeParams, watcher, router, relayer, wallet), "switchboard", SupervisorStrategy.Resume))
     val paymentInitiator = system.actorOf(SimpleSupervisor.props(PaymentInitiator.props(nodeParams.privateKey.publicKey, router, register), "payment-initiator", SupervisorStrategy.Restart))
@@ -168,6 +190,14 @@ class Setup(datadir: File, overrideDefaults: Config = ConfigFactory.empty(), act
 
 }
 
+
+// @formatter:off
+sealed trait Bitcoin
+case class Bitcoind(extendedBitcoinClient: ExtendedBitcoinClient) extends Bitcoin
+case class Bitcoinj(bitcoinjKit: BitcoinjKit) extends Bitcoin
+case class Electrum(electrumClient: ActorRef) extends Bitcoin
+// @formatter:on
+
 case class Kit(nodeParams: NodeParams,
                system: ActorSystem,
                watcher: ActorRef,

eclair-core/src/main/scala/fr/acinq/eclair/UInt64.scala

@@ -31,4 +31,5 @@ object UInt64 {
 
     implicit def longToUint64(l: Long) = UInt64(l)
   }
+
 }

eclair-core/src/main/scala/fr/acinq/eclair/api/JsonSerializers.scala

@@ -1,7 +1,7 @@
 package fr.acinq.eclair.api
 
-import fr.acinq.bitcoin.{BinaryData, Script, ScriptElt, Transaction}
 import fr.acinq.bitcoin.Crypto.{Point, PrivateKey, PublicKey, Scalar}
+import fr.acinq.bitcoin.{BinaryData, Transaction}
 import fr.acinq.eclair.channel.State
 import fr.acinq.eclair.crypto.ShaChain
 import fr.acinq.eclair.transactions.Transactions.TransactionWithInputInfo

eclair-core/src/main/scala/fr/acinq/eclair/api/Service.scala

@@ -77,7 +77,7 @@ trait Service extends Logging {
               import kit._
               val f_res: Future[AnyRef] = req match {
                 case JsonRPCBody(_, _, "getinfo", _) => getInfoResponse
-                case JsonRPCBody(_, _, "connect", JString(host) :: JInt(port) :: JString(nodeId) :: Nil) =>
+                case JsonRPCBody(_, _, "connect", JString(nodeId) :: JString(host) :: JInt(port) :: Nil) =>
                   (switchboard ? NewConnection(PublicKey(nodeId), new InetSocketAddress(host, port.toInt), None)).mapTo[String]
                 case JsonRPCBody(_, _, "open", JString(nodeId) :: JString(host) :: JInt(port) :: JInt(fundingSatoshi) :: JInt(pushMsat) :: options) =>
                   val channelFlags = options match {
@@ -95,7 +95,7 @@ trait Service extends Logging {
                   (router ? 'nodes).mapTo[Iterable[NodeAnnouncement]].map(_.map(_.nodeId))
                 case JsonRPCBody(_, _, "allchannels", _) =>
                   (router ? 'channels).mapTo[Iterable[ChannelAnnouncement]].map(_.map(c => ChannelInfo(c.shortChannelId.toHexString, c.nodeId1, c.nodeId2)))
-                case JsonRPCBody(_,_, "receive", JInt(amountMsat) :: JString(description) :: Nil) =>
+                case JsonRPCBody(_, _, "receive", JInt(amountMsat) :: JString(description) :: Nil) =>
                   (paymentHandler ? ReceivePayment(MilliSatoshi(amountMsat.toLong), description)).mapTo[PaymentRequest].map(PaymentRequest.write)
                 case JsonRPCBody(_, _, "send", JInt(amountMsat) :: JString(paymentHash) :: JString(nodeId) :: Nil) =>
                   (paymentInitiator ? SendPayment(amountMsat.toLong, paymentHash, PublicKey(nodeId))).mapTo[PaymentResult]
@@ -108,7 +108,11 @@ trait Service extends Logging {
                       case (None, JInt(amt) :: Nil) => amt.toLong // amount wasn't specified in request, using custom one
                       case (None, _) => throw new RuntimeException("you need to manually specify an amount for this payment request")
                     }
-                    res <- (paymentInitiator ? SendPayment(amount, req.paymentHash, req.nodeId)).mapTo[PaymentResult]
+                    sendPayment = req.minFinalCltvExpiry match {
+                      case None => SendPayment(amount, req.paymentHash, req.nodeId)
+                      case Some(value) => SendPayment(amount, req.paymentHash, req.nodeId, value)
+                    }
+                    res <- (paymentInitiator ? sendPayment).mapTo[PaymentResult]
                   } yield res
                 case JsonRPCBody(_, _, "close", JString(channelId) :: JString(scriptPubKey) :: Nil) =>
                   getChannel(channelId).flatMap(_ ? CMD_CLOSE(scriptPubKey = Some(scriptPubKey))).mapTo[String]
@@ -116,7 +120,7 @@ trait Service extends Logging {
                   getChannel(channelId).flatMap(_ ? CMD_CLOSE(scriptPubKey = None)).mapTo[String]
                 case JsonRPCBody(_, _, "help", _) =>
                   Future.successful(List(
-                    "connect (host, port, nodeId): connect to another lightning node through a secure connection",
+                    "connect (nodeId, host, port): connect to another lightning node through a secure connection",
                     "open (nodeId, host, port, fundingSatoshi, pushMsat, channelFlags = 0x01): open a channel with another lightning node",
                     "peers: list existing local peers",
                     "channels: list existing local channels",

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/BlockchainEvents.scala

@@ -1,6 +1,7 @@
 package fr.acinq.eclair.blockchain
 
 import fr.acinq.bitcoin.{Block, Transaction}
+import fr.acinq.eclair.blockchain.fee.FeeratesPerKw
 
 /**
   * Created by PM on 24/08/2016.
@@ -14,4 +15,4 @@ case class NewTransaction(tx: Transaction) extends BlockchainEvent
 
 case class CurrentBlockCount(blockCount: Long) extends BlockchainEvent
 
-case class CurrentFeerate(feeratePerKw: Long) extends BlockchainEvent
+case class CurrentFeerates(feeratesPerKw: FeeratesPerKw) extends BlockchainEvent

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/EclairWallet.scala

@@ -0,0 +1,41 @@
+package fr.acinq.eclair.blockchain
+
+import fr.acinq.bitcoin.{BinaryData, Satoshi, Transaction}
+
+import scala.concurrent.Future
+
+/**
+  * Created by PM on 06/07/2017.
+  */
+trait EclairWallet {
+
+  def getBalance: Future[Satoshi]
+
+  def getFinalAddress: Future[String]
+
+  def makeFundingTx(pubkeyScript: BinaryData, amount: Satoshi, feeRatePerKw: Long): Future[MakeFundingTxResponse]
+
+  /**
+    * Committing *must* include publishing the transaction on the network.
+    *
+    * We need to be very careful here, we don't want to consider a commit 'failed' if we are not absolutely sure that the
+    * funding tx won't end up on the blockchain: if that happens and we have cancelled the channel, then we would lose our
+    * funds!
+    *
+    * @param tx
+    * @return true if success
+    *         false IF AND ONLY IF *HAS NOT BEEN PUBLISHED* otherwise funds are at risk!!!
+    */
+  def commit(tx: Transaction): Future[Boolean]
+
+  /**
+    * Cancels this transaction: this probably translates to "release locks on utxos".
+    *
+    * @param tx
+    * @return
+    */
+  def rollback(tx: Transaction): Future[Boolean]
+
+}
+
+final case class MakeFundingTxResponse(fundingTx: Transaction, fundingTxOutputIndex: Int)

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/Types.scala

@@ -1,42 +0,0 @@
-package fr.acinq.eclair.blockchain
-
-import akka.actor.ActorRef
-import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
-import fr.acinq.bitcoin.{BinaryData, Satoshi, Transaction}
-import fr.acinq.eclair.channel.BitcoinEvent
-import fr.acinq.eclair.wire.ChannelAnnouncement
-
-/**
-  * Created by PM on 19/01/2016.
-  */
-
-// @formatter:off
-
-sealed trait Watch {
-  def channel: ActorRef
-  def event: BitcoinEvent
-}
-final case class WatchConfirmed(channel: ActorRef, txId: BinaryData, minDepth: Long, event: BitcoinEvent) extends Watch
-final case class WatchSpent(channel: ActorRef, txId: BinaryData, outputIndex: Int, event: BitcoinEvent) extends Watch
-final case class WatchSpentBasic(channel: ActorRef, txId: BinaryData, outputIndex: Int, event: BitcoinEvent) extends Watch // we use this when we don't care about the spending tx, and we also assume txid already exists
-// TODO: notify me if confirmation number gets below minDepth?
-final case class WatchLost(channel: ActorRef, txId: BinaryData, minDepth: Long, event: BitcoinEvent) extends Watch
-
-trait WatchEvent {
-  def event: BitcoinEvent
-}
-final case class WatchEventConfirmed(event: BitcoinEvent, blockHeight: Int, txIndex: Int) extends WatchEvent
-final case class WatchEventSpent(event: BitcoinEvent, tx: Transaction) extends WatchEvent
-final case class WatchEventSpentBasic(event: BitcoinEvent) extends WatchEvent
-final case class WatchEventLost(event: BitcoinEvent) extends WatchEvent
-final case class WatchEventDoubleSpent(event: BitcoinEvent) extends WatchEvent
-
-/**
-  * Publish the provided tx as soon as possible depending on locktime and csv
-  */
-final case class PublishAsap(tx: Transaction)
-final case class ParallelGetRequest(ann: Seq[ChannelAnnouncement])
-final case class IndividualResult(c: ChannelAnnouncement, tx: Option[Transaction], unspent: Boolean)
-final case class ParallelGetResponse(r: Seq[IndividualResult])
-
-// @formatter:on

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/WatcherTypes.scala

@@ -0,0 +1,67 @@
+package fr.acinq.eclair.blockchain
+
+import akka.actor.ActorRef
+import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.bitcoin.{BinaryData, Script, ScriptWitness, Transaction}
+import fr.acinq.eclair.channel.BitcoinEvent
+import fr.acinq.eclair.wire.ChannelAnnouncement
+
+import scala.util.{Failure, Success, Try}
+
+/**
+  * Created by PM on 19/01/2016.
+  */
+
+// @formatter:off
+
+sealed trait Watch {
+  def channel: ActorRef
+  def event: BitcoinEvent
+}
+// we need a public key script to use bitcoinj or electrum apis
+final case class WatchConfirmed(channel: ActorRef, txId: BinaryData, publicKeyScript: BinaryData, minDepth: Long, event: BitcoinEvent) extends Watch
+object WatchConfirmed {
+  // if we have the entire transaction, we can get the redeemScript from the witness, and re-compute the publicKeyScript
+  // we support both p2pkh and p2wpkh scripts
+  def apply(channel: ActorRef, tx: Transaction, minDepth: Long, event: BitcoinEvent): WatchConfirmed = WatchConfirmed(channel, tx.txid, extractPublicKeyScript(tx.txIn.head.witness), minDepth, event)
+
+  def extractPublicKeyScript(witness: ScriptWitness): BinaryData = Try(PublicKey(witness.stack.last)) match {
+    case Success(pubKey) =>
+      // if last element of the witness is a public key, then this is a p2wpkh
+      Script.write(Script.pay2wpkh(pubKey))
+    case Failure(_) =>
+      // otherwise this is a p2wsh
+      witness.stack.last
+  }
+}
+
+final case class WatchSpent(channel: ActorRef, txId: BinaryData, outputIndex: Int, publicKeyScript: BinaryData, event: BitcoinEvent) extends Watch
+object WatchSpent {
+  // if we have the entire transaction, we can get the publicKeyScript from the relevant output
+  def apply(channel: ActorRef, tx: Transaction, outputIndex: Int, event: BitcoinEvent): WatchSpent = WatchSpent(channel, tx.txid, outputIndex, tx.txOut(outputIndex).publicKeyScript, event)
+}
+final case class WatchSpentBasic(channel: ActorRef, txId: BinaryData, outputIndex: Int, publicKeyScript: BinaryData, event: BitcoinEvent) extends Watch // we use this when we don't care about the spending tx, and we also assume txid already exists
+object WatchSpentBasic {
+  // if we have the entire transaction, we can get the publicKeyScript from the relevant output
+  def apply(channel: ActorRef, tx: Transaction, outputIndex: Int, event: BitcoinEvent): WatchSpentBasic = WatchSpentBasic(channel, tx.txid, outputIndex, tx.txOut(outputIndex).publicKeyScript, event)
+}
+// TODO: notify me if confirmation number gets below minDepth?
+final case class WatchLost(channel: ActorRef, txId: BinaryData, minDepth: Long, event: BitcoinEvent) extends Watch
+
+trait WatchEvent {
+  def event: BitcoinEvent
+}
+final case class WatchEventConfirmed(event: BitcoinEvent, blockHeight: Int, txIndex: Int) extends WatchEvent
+final case class WatchEventSpent(event: BitcoinEvent, tx: Transaction) extends WatchEvent
+final case class WatchEventSpentBasic(event: BitcoinEvent) extends WatchEvent
+final case class WatchEventLost(event: BitcoinEvent) extends WatchEvent
+
+/**
+  * Publish the provided tx as soon as possible depending on locktime and csv
+  */
+final case class PublishAsap(tx: Transaction)
+final case class ParallelGetRequest(ann: Seq[ChannelAnnouncement])
+final case class IndividualResult(c: ChannelAnnouncement, tx: Option[Transaction], unspent: Boolean)
+final case class ParallelGetResponse(r: Seq[IndividualResult])
+
+// @formatter:on

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreWallet.scala

@@ -1,11 +1,11 @@
-package fr.acinq.eclair.blockchain.wallet
+package fr.acinq.eclair.blockchain.bitcoind
 
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
-import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
-import fr.acinq.bitcoin.{Base58Check, BinaryData, Crypto, OP_PUSHDATA, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptFlags, ScriptWitness, SigVersion, Transaction, TxIn, TxOut}
+import fr.acinq.bitcoin.Crypto.PrivateKey
+import fr.acinq.bitcoin.{Base58Check, BinaryData, OP_PUSHDATA, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptFlags, ScriptWitness, SigVersion, Transaction, TxIn, TxOut}
 import fr.acinq.eclair.blockchain._
-import fr.acinq.eclair.blockchain.rpc.BitcoinJsonRPCClient
-import fr.acinq.eclair.channel.{BITCOIN_INPUT_SPENT, BITCOIN_TX_CONFIRMED, Helpers}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, JsonRPCError}
+import fr.acinq.eclair.channel.{BITCOIN_OUTPUT_SPENT, BITCOIN_TX_CONFIRMED}
 import fr.acinq.eclair.transactions.Transactions
 import grizzled.slf4j.Logging
 import org.json4s.JsonAST.{JBool, JDouble, JInt, JString}
@@ -14,6 +14,14 @@ import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future, Promise}
 
 /**
+  * Due to bitcoin-core wallet not fully supporting segwit txes yet, our current scheme is:
+  * utxos <- parent-tx <- funding-tx
+  *
+  * With:
+  *   - utxos may be non-segwit
+  *   - parent-tx pays to a p2wpkh segwit output
+  *   - funding-tx is a segwit tx
+  *
   * Created by PM on 06/07/2017.
   */
 class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(implicit system: ActorSystem, ec: ExecutionContext) extends EclairWallet with Logging {
@@ -31,8 +39,8 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
 
   case class MakeFundingTxResponseWithParent(parentTx: Transaction, fundingTx: Transaction, fundingTxOutputIndex: Int, priv: PrivateKey)
 
-  def fundTransaction(hex: String): Future[FundTransactionResponse] = {
-    rpcClient.invoke("fundrawtransaction", hex).map(json => {
+  def fundTransaction(hex: String, lockUnspents: Boolean): Future[FundTransactionResponse] = {
+    rpcClient.invoke("fundrawtransaction", hex, BitcoinCoreWallet.Options(lockUnspents)).map(json => {
       val JString(hex) = json \ "hex"
       val JInt(changepos) = json \ "changepos"
       val JDouble(fee) = json \ "fee"
@@ -40,8 +48,8 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
     })
   }
 
-  def fundTransaction(tx: Transaction): Future[FundTransactionResponse] =
-    fundTransaction(Transaction.write(tx).toString())
+  def fundTransaction(tx: Transaction, lockUnspents: Boolean): Future[FundTransactionResponse] =
+    fundTransaction(Transaction.write(tx).toString(), lockUnspents)
 
   def signTransaction(hex: String): Future[SignTransactionResponse] =
     rpcClient.invoke("signrawtransaction", hex).map(json => {
@@ -53,6 +61,21 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
   def signTransaction(tx: Transaction): Future[SignTransactionResponse] =
     signTransaction(Transaction.write(tx).toString())
 
+  def getTransaction(txid: BinaryData): Future[Transaction] = {
+    rpcClient.invoke("getrawtransaction", txid.toString()).map(json => {
+      val JString(hex) = json
+      Transaction.read(hex)
+    })
+  }
+
+  def publishTransaction(tx: Transaction)(implicit ec: ExecutionContext): Future[String] =
+    publishTransaction(Transaction.write(tx).toString())
+
+  def publishTransaction(hex: String)(implicit ec: ExecutionContext): Future[String] =
+    rpcClient.invoke("sendrawtransaction", hex) collect {
+      case JString(txid) => txid
+    }
+
   /**
     *
     * @param fundingTxResponse a funding tx response
@@ -97,7 +120,7 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
 
   def makeParentAndFundingTx(pubkeyScript: BinaryData, amount: Satoshi, feeRatePerKw: Long): Future[MakeFundingTxResponseWithParent] =
     for {
-    // ask for a new address and the corresponding private key
+      // ask for a new address and the corresponding private key
       JString(address) <- rpcClient.invoke("getnewaddress")
       JString(wif) <- rpcClient.invoke("dumpprivkey", address)
       JString(segwitAddress) <- rpcClient.invoke("addwitnessaddress", address)
@@ -111,7 +134,7 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
         txIn = Nil,
         txOut = TxOut(amount + parentFee, Script.pay2sh(Script.pay2wpkh(pub))) :: Nil,
         lockTime = 0L)
-      FundTransactionResponse(unsignedParentTx, _, _) <- fundTransaction(partialParentTx)
+      FundTransactionResponse(unsignedParentTx, _, _) <- fundTransaction(partialParentTx, lockUnspents = true)
       // this is the first tx that we will publish, a standard tx which send money to our p2wpkh address
       SignTransactionResponse(parentTx, true) <- signTransaction(unsignedParentTx)
       // now we create the funding tx
@@ -137,16 +160,18 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
     val promise = Promise[MakeFundingTxResponse]()
     (for {
       fundingTxResponse@MakeFundingTxResponseWithParent(parentTx, _, _, _) <- makeParentAndFundingTx(pubkeyScript, amount, feeRatePerKw)
+      input0 = parentTx.txIn.head
+      parentOfParentTx <- getTransaction(input0.outPoint.txid)
       _ = logger.debug(s"built parentTxid=${parentTx.txid}, initializing temporary actor")
       tempActor = system.actorOf(Props(new Actor {
         override def receive: Receive = {
-          case WatchEventSpent(BITCOIN_INPUT_SPENT(parentTx), spendingTx) =>
+          case WatchEventSpent(BITCOIN_OUTPUT_SPENT, spendingTx) =>
             if (parentTx.txid != spendingTx.txid) {
               // an input of our parent tx was spent by a tx that we're not aware of (i.e. a malleated version of our parent tx)
               // set a new watch; if it is confirmed, we'll use it as the new parent for our funding tx
               logger.warn(s"parent tx has been malleated: originalParentTxid=${parentTx.txid} malleated=${spendingTx.txid}")
             }
-            watcher ! WatchConfirmed(self, spendingTx.txid, minDepth = 1, BITCOIN_TX_CONFIRMED(spendingTx))
+            watcher ! WatchConfirmed(self, spendingTx.txid, spendingTx.txOut(0).publicKeyScript, minDepth = 1, BITCOIN_TX_CONFIRMED(spendingTx))
 
           case WatchEventConfirmed(BITCOIN_TX_CONFIRMED(tx), _, _) =>
             // a potential parent for our funding tx has been confirmed, let's update our funding tx
@@ -155,8 +180,7 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
         }
       }))
       // we watch the first input of the parent tx, so that we can detect when it is spent by a malleated avatar
-      input0 = parentTx.txIn.head
-      _ = watcher ! WatchSpent(tempActor, input0.outPoint.txid, input0.outPoint.index.toInt, BITCOIN_INPUT_SPENT(parentTx))
+      _ = watcher ! WatchSpent(tempActor, input0.outPoint.txid, input0.outPoint.index.toInt, parentOfParentTx.txOut(input0.outPoint.index.toInt).publicKeyScript, BITCOIN_OUTPUT_SPENT)
       // and we publish the parent tx
       _ = logger.info(s"publishing parent tx: txid=${parentTx.txid} tx=${Transaction.write(parentTx)}")
       // we use a small delay so that we are sure Publish doesn't race with WatchSpent (which is ok but generates unnecessary warnings)
@@ -167,11 +191,24 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient, watcher: ActorRef)(impl
     promise.future
   }
 
+  override def commit(tx: Transaction): Future[Boolean] = publishTransaction(tx)
+    .map(_ => true) // if bitcoind says OK, then we consider the tx succesfully published
+    .recoverWith { case JsonRPCError(_) => getTransaction(tx.txid).map(_ => true).recover { case _ => false } } // if we get a parseable error from bitcoind AND the tx is NOT in the mempool/blockchain, then we consider that the tx was not published
+    .recover { case _ => true } // in all other cases we consider that the tx has been published
+
+
   /**
-    * We don't manage double spends yet
+    * We currently only put a lock on the parent tx inputs, and we publish the parent tx immediately so there is nothing
+    * to do here.
+    *
     * @param tx
     * @return
     */
-  override def commit(tx: Transaction): Future[Boolean] = Future.successful(true)
-
+  override def rollback(tx: Transaction): Future[Boolean] = Future.successful(true)
 }
+
+object BitcoinCoreWallet {
+
+  case class Options(lockUnspents: Boolean)
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala

@@ -1,14 +1,15 @@
-package fr.acinq.eclair.blockchain
+package fr.acinq.eclair.blockchain.bitcoind
 
 import java.util.concurrent.Executors
 
 import akka.actor.{Actor, ActorLogging, Cancellable, Props, Terminated}
 import akka.pattern.pipe
 import fr.acinq.bitcoin._
-import fr.acinq.eclair.blockchain.rpc.ExtendedBitcoinClient
+import fr.acinq.eclair.Globals
+import fr.acinq.eclair.blockchain._
+import fr.acinq.eclair.blockchain.bitcoind.rpc.ExtendedBitcoinClient
 import fr.acinq.eclair.channel.BITCOIN_PARENT_TX_CONFIRMED
 import fr.acinq.eclair.transactions.Scripts
-import fr.acinq.eclair.{Globals, NodeParams}
 
 import scala.collection.SortedMap
 import scala.concurrent.duration._
@@ -23,10 +24,12 @@ import scala.util.Try
   */
 class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext = ExecutionContext.global) extends Actor with ActorLogging {
 
+  import ZmqWatcher.TickNewBlock
+
   context.system.eventStream.subscribe(self, classOf[BlockchainEvent])
 
   // this is to initialize block count
-  self ! 'tick
+  self ! TickNewBlock
 
   case class TriggerEvent(w: Watch, e: WatchEvent)
 
@@ -34,27 +37,25 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
 
   def watching(watches: Set[Watch], block2tx: SortedMap[Long, Seq[Transaction]], nextTick: Option[Cancellable]): Receive = {
 
-    case hint: Hint => {}
-
     case NewTransaction(tx) =>
       //log.debug(s"analyzing txid=${tx.txid} tx=${Transaction.write(tx)}")
       watches.collect {
-        case w@WatchSpentBasic(_, txid, outputIndex, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
+        case w@WatchSpentBasic(_, txid, outputIndex, _, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
           self ! TriggerEvent(w, WatchEventSpentBasic(event))
-        case w@WatchSpent(_, txid, outputIndex, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
+        case w@WatchSpent(_, txid, outputIndex, _, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
           self ! TriggerEvent(w, WatchEventSpent(event, tx))
       }
 
     case NewBlock(block) =>
       // using a Try because in tests we generate fake blocks
       log.debug(s"received blockid=${Try(block.blockId).getOrElse(BinaryData(""))}")
-      nextTick.map(_.cancel()) // this may fail or succeed, worse case scenario we will have two 'ticks in a row (no big deal)
+      nextTick.map(_.cancel()) // this may fail or succeed, worse case scenario we will have two ticks in a row (no big deal)
       log.debug(s"scheduling a new task to check on tx confirmations")
       // we do this to avoid herd effects in testing when generating a lots of blocks in a row
-      val task = context.system.scheduler.scheduleOnce(2 seconds, self, 'tick)
+      val task = context.system.scheduler.scheduleOnce(2 seconds, self, TickNewBlock)
       context become watching(watches, block2tx, Some(task))
 
-    case 'tick =>
+    case TickNewBlock =>
       client.getBlockCount.map {
         case count =>
           log.debug(s"setting blockCount=$count")
@@ -71,7 +72,7 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
       }*/
       // TODO: beware of the herd effect
       watches.collect {
-        case w@WatchConfirmed(_, txId, minDepth, event) =>
+        case w@WatchConfirmed(_, txId, _, minDepth, event) =>
           log.debug(s"checking confirmations of txid=$txId")
           client.getTxConfirmations(txId.toString).map {
             case Some(confirmations) if confirmations >= minDepth =>
@@ -105,10 +106,11 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
         require(tx.txIn.size == 1, s"watcher only supports tx with 1 input, this tx has ${tx.txIn.size} inputs")
         val parentTxid = tx.txIn(0).outPoint.txid
         log.info(s"txid=${tx.txid} has a relative timeout of $csvTimeout blocks, watching parenttxid=$parentTxid tx=${Transaction.write(tx)}")
-        self ! WatchConfirmed(self, parentTxid, minDepth = 1, BITCOIN_PARENT_TX_CONFIRMED(tx))
+        val parentPublicKey = fr.acinq.bitcoin.Script.write(fr.acinq.bitcoin.Script.pay2wsh(tx.txIn.head.witness.stack.last))
+        self ! WatchConfirmed(self, parentTxid, parentPublicKey, minDepth = 1, BITCOIN_PARENT_TX_CONFIRMED(tx))
       } else if (cltvTimeout > blockCount) {
         log.info(s"delaying publication of txid=${tx.txid} until block=$cltvTimeout (curblock=$blockCount)")
-        val block2tx1 = block2tx.updated(cltvTimeout, tx +: block2tx.getOrElse(cltvTimeout, Seq.empty[Transaction]))
+        val block2tx1 = block2tx.updated(cltvTimeout, block2tx.getOrElse(cltvTimeout, Seq.empty[Transaction]) :+ tx)
         context.become(watching(watches, block2tx1, None))
       } else publish(tx)
 
@@ -119,7 +121,7 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
       val absTimeout = blockHeight + csvTimeout
       if (absTimeout > blockCount) {
         log.info(s"delaying publication of txid=${tx.txid} until block=$absTimeout (curblock=$blockCount)")
-        val block2tx1 = block2tx.updated(absTimeout, tx +: block2tx.getOrElse(absTimeout, Seq.empty[Transaction]))
+        val block2tx1 = block2tx.updated(absTimeout, block2tx.getOrElse(absTimeout, Seq.empty[Transaction]) :+ tx)
         context.become(watching(watches, block2tx1, None))
       } else publish(tx)
 
@@ -136,7 +138,7 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
 
   def addWatch(w: Watch, watches: Set[Watch], block2tx: SortedMap[Long, Seq[Transaction]]) = {
     w match {
-      case WatchSpentBasic(_, txid, outputIndex, _) =>
+      case WatchSpentBasic(_, txid, outputIndex, _, _) =>
         // not: we assume parent tx was published, we just need to make sure this particular output has not been spent
         client.isTransactionOuputSpendable(txid.toString(), outputIndex, true).collect {
           case false =>
@@ -144,7 +146,7 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
             self ! TriggerEvent(w, WatchEventSpentBasic(w.event))
         }
 
-      case w@WatchSpent(_, txid, outputIndex, _) =>
+      case w@WatchSpent(_, txid, outputIndex, _, _) =>
         // first let's see if the parent tx was published or not
         client.getTxConfirmations(txid.toString()).collect {
           case Some(_) =>
@@ -171,7 +173,7 @@ class ZmqWatcher(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext =
             }
         }
 
-      case w: WatchConfirmed => self ! 'tick
+      case w: WatchConfirmed => self ! TickNewBlock
 
       case w => log.warning(s"ignoring $w (not implemented)")
     }
@@ -202,4 +204,6 @@ object ZmqWatcher {
 
   def props(client: ExtendedBitcoinClient)(implicit ec: ExecutionContext = ExecutionContext.global) = Props(new ZmqWatcher(client)(ec))
 
+  case object TickNewBlock
+
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinJsonRPCClient.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain.rpc
+package fr.acinq.eclair.blockchain.bitcoind.rpc
 
 import java.io.IOException
 
@@ -8,12 +8,15 @@ import akka.http.scaladsl.marshalling.Marshal
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.model.headers.{Authorization, BasicHttpCredentials}
 import akka.http.scaladsl.unmarshalling.Unmarshal
-import akka.stream.ActorMaterializer
+import akka.stream.{ActorMaterializer, OverflowStrategy, QueueOfferResult}
+import akka.stream.scaladsl.{Keep, Sink, Source}
 import de.heikoseeberger.akkahttpjson4s.Json4sSupport._
 import org.json4s.JsonAST.JValue
 import org.json4s.{DefaultFormats, jackson}
 
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext, Future, Promise}
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.util.{Failure, Success}
 
 // @formatter:off
 case class JsonRPCRequest(jsonrpc: String = "1.0", id: String = "scala-client", method: String, params: Seq[Any])
@@ -26,16 +29,35 @@ class BitcoinJsonRPCClient(user: String, password: String, host: String = "127.0
 
   val scheme = if (ssl) "https" else "http"
   val uri = Uri(s"$scheme://$host:$port")
-
-  implicit val materializer = ActorMaterializer()
-  val httpClient = Http(system)
   implicit val serialization = jackson.Serialization
   implicit val formats = DefaultFormats
 
+  implicit val materializer = ActorMaterializer()
+  val httpClientFlow = Http().cachedHostConnectionPool[Promise[HttpResponse]](host, port)
+
+  val queueSize = 512
+  val queue = Source.queue[(HttpRequest, Promise[HttpResponse])](queueSize, OverflowStrategy.dropNew)
+      .via(httpClientFlow)
+      .toMat(Sink.foreach({
+        case ((Success(resp), p)) => p.success(resp)
+        case ((Failure(e), p))    => p.failure(e)
+      }))(Keep.left)
+      .run()
+
+  def queueRequest(request: HttpRequest): Future[HttpResponse] = {
+    val responsePromise = Promise[HttpResponse]()
+    queue.offer(request -> responsePromise).flatMap {
+      case QueueOfferResult.Enqueued    => responsePromise.future
+      case QueueOfferResult.Dropped     => Future.failed(new RuntimeException("Queue overflowed. Try again later."))
+      case QueueOfferResult.Failure(ex) => Future.failed(ex)
+      case QueueOfferResult.QueueClosed => Future.failed(new RuntimeException("Queue was closed (pool shut down) while running the request. Try again later."))
+    }
+  }
+
   def invoke(method: String, params: Any*)(implicit ec: ExecutionContext): Future[JValue] =
     for {
       entity <- Marshal(JsonRPCRequest(method = method, params = params)).to[RequestEntity]
-      httpRes <- httpClient.singleRequest(HttpRequest(uri = uri, method = HttpMethods.POST).addHeader(Authorization(BasicHttpCredentials(user, password))).withEntity(entity))
+      httpRes <- queueRequest(HttpRequest(uri = "/", method = HttpMethods.POST).addHeader(Authorization(BasicHttpCredentials(user, password))).withEntity(entity))
       jsonRpcRes <- Unmarshal(httpRes).to[JsonRPCResponse].map {
         case JsonRPCResponse(_, Some(error), _) => throw JsonRPCError(error)
         case o => o
@@ -47,7 +69,7 @@ class BitcoinJsonRPCClient(user: String, password: String, host: String = "127.0
   def invoke(request: Seq[(String, Seq[Any])])(implicit ec: ExecutionContext): Future[Seq[JValue]] =
     for {
       entity <- Marshal(request.map(r => JsonRPCRequest(method = r._1, params = r._2))).to[RequestEntity]
-      httpRes <- httpClient.singleRequest(HttpRequest(uri = uri, method = HttpMethods.POST).addHeader(Authorization(BasicHttpCredentials(user, password))).withEntity(entity))
+      httpRes <- queueRequest(HttpRequest(uri = "/", method = HttpMethods.POST).addHeader(Authorization(BasicHttpCredentials(user, password))).withEntity(entity))
       jsonRpcRes <- Unmarshal(httpRes).to[Seq[JsonRPCResponse]].map {
         //case JsonRPCResponse(_, Some(error), _) => throw JsonRPCError(error)
         case o => o

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/ExtendedBitcoinClient.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain.rpc
+package fr.acinq.eclair.blockchain.bitcoind.rpc
 
 import fr.acinq.bitcoin._
 import fr.acinq.eclair.blockchain.{IndividualResult, ParallelGetResponse}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/zmq/ZMQActor.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain.zmq
+package fr.acinq.eclair.blockchain.bitcoind.zmq
 
 import akka.actor.{Actor, ActorLogging}
 import fr.acinq.bitcoin.{Block, Transaction}
@@ -15,6 +15,8 @@ import scala.util.Try
   */
 class ZMQActor(address: String, connected: Option[Promise[Boolean]] = None) extends Actor with ActorLogging {
 
+  import ZMQActor._
+
   val ctx = new ZContext
 
   val subscriber = ctx.createSocket(ZMQ.SUB)
@@ -75,3 +77,13 @@ class ZMQActor(address: String, connected: Option[Promise[Boolean]] = None) exte
   }
 
 }
+
+object ZMQActor {
+
+  // @formatter:off
+  sealed trait ZMQEvent
+  case object ZMQConnected extends ZMQEvent
+  case object ZMQDisconnected extends ZMQEvent
+  // @formatter:on
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoinj/BitcoinjKit.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain.spv
+package fr.acinq.eclair.blockchain.bitcoinj
 
 import java.io.File
 import java.net.InetSocketAddress
@@ -7,8 +7,8 @@ import akka.actor.ActorSystem
 import com.google.common.util.concurrent.{FutureCallback, Futures}
 import fr.acinq.bitcoin.Transaction
 import fr.acinq.eclair.Globals
-import fr.acinq.eclair.blockchain.spv.BitcoinjKit._
-import fr.acinq.eclair.blockchain.{CurrentBlockCount, NewConfidenceLevel}
+import fr.acinq.eclair.blockchain.CurrentBlockCount
+import fr.acinq.eclair.blockchain.bitcoinj.BitcoinjKit._
 import grizzled.slf4j.Logging
 import org.bitcoinj.core.TransactionConfidence.ConfidenceType
 import org.bitcoinj.core.listeners._
@@ -41,8 +41,8 @@ class BitcoinjKit(chain: String, datadir: File, staticPeers: List[InetSocketAddr
   val atCurrentHeight = atCurrentHeightPromise.future
 
   // tells us when we are at current block height
-//  private val syncedPromise = Promise[Boolean]()
-//  val synced = syncedPromise.future
+  //  private val syncedPromise = Promise[Boolean]()
+  //  val synced = syncedPromise.future
 
   private def updateBlockCount(blockCount: Int) = {
     // when synchronizing we don't want to advertise previous blocks
@@ -61,13 +61,13 @@ class BitcoinjKit(chain: String, datadir: File, staticPeers: List[InetSocketAddr
     peerGroup().setMinRequiredProtocolVersion(70015) // bitcoin core 0.13
     wallet().watchMode = true
 
-//    setDownloadListener(new DownloadProgressTracker {
-//      override def doneDownload(): Unit = {
-//        super.doneDownload()
-//        // may be called multiple times
-//        syncedPromise.trySuccess(true)
-//      }
-//    })
+    //    setDownloadListener(new DownloadProgressTracker {
+    //      override def doneDownload(): Unit = {
+    //        super.doneDownload()
+    //        // may be called multiple times
+    //        syncedPromise.trySuccess(true)
+    //      }
+    //    })
 
     // we set the blockcount to the previous stored block height
     updateBlockCount(chain().getBestChainHeight)

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoinj/BitcoinjWallet.scala

@@ -1,6 +1,7 @@
-package fr.acinq.eclair.blockchain.wallet
+package fr.acinq.eclair.blockchain.bitcoinj
 
 import fr.acinq.bitcoin.{BinaryData, Satoshi, Transaction}
+import fr.acinq.eclair.blockchain.{EclairWallet, MakeFundingTxResponse}
 import grizzled.slf4j.Logging
 import org.bitcoinj.core.{Coin, Context, Transaction => BitcoinjTransaction}
 import org.bitcoinj.script.Script
@@ -56,4 +57,12 @@ class BitcoinjWallet(val fWallet: Future[Wallet])(implicit ec: ExecutionContext)
       _ = logger.info(s"commit txid=${tx.txid} result=$canCommit")
     } yield canCommit
   }
+
+  /**
+    * There are no locks on bitcoinj, this is a no-op
+    *
+    * @param tx
+    * @return
+    */
+  override def rollback(tx: Transaction) = Future.successful(true)
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoinj/BitcoinjWatcher.scala

@@ -1,25 +1,23 @@
-package fr.acinq.eclair.blockchain
+package fr.acinq.eclair.blockchain.bitcoinj
 
 import akka.actor.{Actor, ActorLogging, Props, Terminated}
 import com.google.common.collect.ImmutableList
 import com.google.common.util.concurrent.{FutureCallback, Futures}
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.Script.{pay2wsh, write}
-import fr.acinq.bitcoin.{Satoshi, Transaction, TxIn, TxOut}
-import fr.acinq.eclair.{Globals, fromShortId}
+import fr.acinq.bitcoin.{BinaryData, Satoshi, Transaction, TxIn, TxOut}
+import fr.acinq.eclair.blockchain._
 import fr.acinq.eclair.channel.BITCOIN_PARENT_TX_CONFIRMED
 import fr.acinq.eclair.transactions.Scripts
+import fr.acinq.eclair.{Globals, fromShortId}
 import org.bitcoinj.core.{Context, Transaction => BitcoinjTransaction}
 import org.bitcoinj.kits.WalletAppKit
 import org.bitcoinj.script.Script
 
 import scala.collection.SortedMap
 import scala.concurrent.ExecutionContext
-import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
-final case class Hint(script: Script)
-
 final case class NewConfidenceLevel(tx: Transaction, blockHeight: Int, confirmations: Int) extends BlockchainEvent
 
 /**
@@ -28,13 +26,11 @@ final case class NewConfidenceLevel(tx: Transaction, blockHeight: Int, confirmat
   * - also uses bitcoin-core rpc api, most notably for tx confirmation count and blockcount (because reorgs)
   * Created by PM on 21/02/2016.
   */
-class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = ExecutionContext.global) extends Actor with ActorLogging {
+class BitcoinjWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = ExecutionContext.global) extends Actor with ActorLogging {
 
   context.system.eventStream.subscribe(self, classOf[BlockchainEvent])
   context.system.eventStream.subscribe(self, classOf[NewConfidenceLevel])
 
-  context.system.scheduler.schedule(10 seconds, 1 minute, self, 'tick)
-
   val broadcaster = context.actorOf(Props(new Broadcaster(kit: WalletAppKit)), name = "broadcaster")
 
   case class TriggerEvent(w: Watch, e: WatchEvent)
@@ -46,15 +42,12 @@ class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = Executio
     case event@NewConfidenceLevel(tx, blockHeight, confirmations) =>
       log.debug(s"analyzing txid=${tx.txid} confirmations=$confirmations tx=${Transaction.write(tx)}")
       watches.collect {
-        case w@WatchSpentBasic(_, txid, outputIndex, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
+        case w@WatchSpentBasic(_, txid, outputIndex, _, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
           self ! TriggerEvent(w, WatchEventSpentBasic(event))
-        case w@WatchSpent(_, txid, outputIndex, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
+        case w@WatchSpent(_, txid, outputIndex, _, event) if tx.txIn.exists(i => i.outPoint.txid == txid && i.outPoint.index == outputIndex) =>
           self ! TriggerEvent(w, WatchEventSpent(event, tx))
-        case w@WatchConfirmed(_, txId, minDepth, event) if txId == tx.txid && confirmations >= minDepth =>
+        case w@WatchConfirmed(_, txId, _, minDepth, event) if txId == tx.txid && confirmations >= minDepth =>
           self ! TriggerEvent(w, WatchEventConfirmed(event, blockHeight, 0))
-        case w@WatchConfirmed(_, txId, minDepth, event) if txId == tx.txid && confirmations == -1 =>
-          // the transaction watched was overriden by a competing tx
-          self ! TriggerEvent(w, WatchEventDoubleSpent(event))
       }
       context become watching(watches, block2tx, oldEvents.filterNot(_.tx.txid == tx.txid) :+ event, sent)
 
@@ -72,15 +65,13 @@ class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = Executio
       context.become(watching(watches, block2tx -- toPublish.keys, oldEvents, sent))
     }
 
-    case hint: Hint => {
-      Context.propagate(kit.wallet.getContext)
-      val script = hint.script
-      // set creation time to 2017/09/01, so bitcoinj can still use its checkpoints optimizations
-      script.setCreationTimeSeconds(1501538400L) // 2017-09-01
-      kit.wallet().addWatchedScripts(ImmutableList.of(script))
-    }
-
     case w: Watch if !watches.contains(w) =>
+      w match {
+        case w: WatchConfirmed => addHint(w.publicKeyScript)
+        case w: WatchSpent => addHint(w.publicKeyScript)
+        case w: WatchSpentBasic => addHint(w.publicKeyScript)
+        case _ => ()
+      }
       log.debug(s"adding watch $w for $sender")
       log.info(s"resending ${oldEvents.size} events!")
       oldEvents.foreach(self ! _)
@@ -95,10 +86,11 @@ class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = Executio
         require(tx.txIn.size == 1, s"watcher only supports tx with 1 input, this tx has ${tx.txIn.size} inputs")
         val parentTxid = tx.txIn(0).outPoint.txid
         log.info(s"txid=${tx.txid} has a relative timeout of $csvTimeout blocks, watching parenttxid=$parentTxid tx=${Transaction.write(tx)}")
-        self ! WatchConfirmed(self, parentTxid, minDepth = 1, BITCOIN_PARENT_TX_CONFIRMED(tx))
+        val parentPublicKey = fr.acinq.bitcoin.Script.write(fr.acinq.bitcoin.Script.pay2wsh(tx.txIn.head.witness.stack.last))
+        self ! WatchConfirmed(self, parentTxid, parentPublicKey, minDepth = 1, BITCOIN_PARENT_TX_CONFIRMED(tx))
       } else if (cltvTimeout > blockCount) {
         log.info(s"delaying publication of txid=${tx.txid} until block=$cltvTimeout (curblock=$blockCount)")
-        val block2tx1 = block2tx.updated(cltvTimeout, tx +: block2tx.getOrElse(cltvTimeout, Seq.empty[Transaction]))
+        val block2tx1 = block2tx.updated(cltvTimeout, block2tx.getOrElse(cltvTimeout, Seq.empty[Transaction]) :+ tx)
         context.become(watching(watches, block2tx1, oldEvents, sent))
       } else publish(tx)
 
@@ -109,7 +101,7 @@ class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = Executio
       val absTimeout = blockHeight + csvTimeout
       if (absTimeout > blockCount) {
         log.info(s"delaying publication of txid=${tx.txid} until block=$absTimeout (curblock=$blockCount)")
-        val block2tx1 = block2tx.updated(absTimeout, tx +: block2tx.getOrElse(absTimeout, Seq.empty[Transaction]))
+        val block2tx1 = block2tx.updated(absTimeout, block2tx.getOrElse(absTimeout, Seq.empty[Transaction]) :+ tx)
         context.become(watching(watches, block2tx1, oldEvents, sent))
       } else publish(tx)
 
@@ -135,13 +127,27 @@ class SpvWatcher(val kit: WalletAppKit)(implicit ec: ExecutionContext = Executio
 
   }
 
+  /**
+    * Bitcoinj needs hints to be able to detect transactions
+    *
+    * @param pubkeyScript
+    * @return
+    */
+  def addHint(pubkeyScript: BinaryData) = {
+    Context.propagate(kit.wallet.getContext)
+    val script = new Script(pubkeyScript)
+    // set creation time to 2017/09/01, so bitcoinj can still use its checkpoints optimizations
+    script.setCreationTimeSeconds(1501538400L) // 2017-09-01
+    kit.wallet().addWatchedScripts(ImmutableList.of(script))
+  }
+
   def publish(tx: Transaction): Unit = broadcaster ! tx
 
 }
 
-object SpvWatcher {
+object BitcoinjWatcher {
 
-  def props(kit: WalletAppKit)(implicit ec: ExecutionContext = ExecutionContext.global) = Props(new SpvWatcher(kit)(ec))
+  def props(kit: WalletAppKit)(implicit ec: ExecutionContext = ExecutionContext.global) = Props(new BitcoinjWatcher(kit)(ec))
 
 }
 
@@ -171,6 +177,7 @@ class Broadcaster(kit: WalletAppKit) extends Actor with ActorLogging {
   }
 
   case class BroadcastResult(tx: Transaction, result: Try[Boolean])
+
   def broadcast(tx: Transaction) = {
     Context.propagate(kit.wallet().getContext)
     val bitcoinjTx = new org.bitcoinj.core.Transaction(kit.wallet().getParams, Transaction.write(tx))
@@ -183,5 +190,4 @@ class Broadcaster(kit: WalletAppKit) extends Actor with ActorLogging {
   }
 
 
-
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumClient.scala

@@ -0,0 +1,488 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import java.io.InputStream
+import java.net.InetSocketAddress
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Stash, Terminated}
+import akka.io.{IO, Tcp}
+import akka.util.ByteString
+import fr.acinq.bitcoin._
+import fr.acinq.eclair.Globals
+import fr.acinq.eclair.blockchain.CurrentBlockCount
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{Error, JsonRPCRequest, JsonRPCResponse}
+import org.json4s.JsonAST._
+import org.json4s.jackson.JsonMethods
+import org.json4s.{DefaultFormats, JInt, JLong, JString}
+import org.spongycastle.util.encoders.Hex
+
+import scala.annotation.tailrec
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.util.Random
+
+class ElectrumClient(serverAddresses: Seq[InetSocketAddress]) extends Actor with Stash with ActorLogging {
+
+  import ElectrumClient._
+  import context.system
+
+  implicit val formats = DefaultFormats
+
+  val newline = "\n"
+  val connectionFailures = collection.mutable.HashMap.empty[InetSocketAddress, Long]
+
+  val version = ServerVersion("2.1.7", "1.1")
+  // we need to regularly send a ping in order not to get disconnected
+  context.system.scheduler.schedule(30 seconds, 30 seconds, self, version)
+
+  override def unhandled(message: Any): Unit = {
+    message match {
+      case _: Tcp.ConnectionClosed =>
+        val nextAddress = nextPeer()
+        log.warning(s"connection failed, trying $nextAddress")
+        self ! Tcp.Connect(nextAddress)
+        statusListeners.map(_ ! ElectrumDisconnected)
+        context.system.eventStream.publish(ElectrumDisconnected)
+        context become disconnected
+
+      case Terminated(deadActor) =>
+        val removeMe = addressSubscriptions collect {
+          case (address, actor) if actor == deadActor => address
+        }
+        addressSubscriptions --= removeMe
+
+        val removeMe1 = scriptHashSubscriptions collect {
+          case (scriptHash, actor) if actor == deadActor => scriptHash
+        }
+        scriptHashSubscriptions --= removeMe1
+        statusListeners -= deadActor
+        headerSubscriptions -= deadActor
+
+      case _: ServerVersion => () // we only handle this when connected
+
+      case _: ServerVersionResponse => () // we just ignore these messages, they are used as pings
+
+      case _ => log.warning(s"unhandled $message")
+    }
+  }
+
+  val statusListeners = collection.mutable.HashSet.empty[ActorRef]
+
+  def send(connection: ActorRef, request: JsonRPCRequest): Unit = {
+    import org.json4s.JsonDSL._
+    import org.json4s._
+    import org.json4s.jackson.JsonMethods._
+
+    log.debug(s"sending $request")
+    val json = ("method" -> request.method) ~ ("params" -> request.params.map {
+      case s: String => new JString(s)
+      case b: BinaryData => new JString(b.toString())
+      case t: Int => new JInt(t)
+      case t: Long => new JLong(t)
+      case t: Double => new JDouble(t)
+    }) ~ ("id" -> request.id) ~ ("jsonrpc" -> request.jsonrpc)
+    val serialized = compact(render(json))
+    val bytes = (serialized + newline).getBytes
+    connection ! Tcp.Write(ByteString.fromArray(bytes))
+  }
+
+  private def nextPeer() = {
+    val nextPos = Random.nextInt(serverAddresses.size)
+    serverAddresses(nextPos)
+  }
+
+  private def updateBlockCount(blockCount: Long) = {
+    // when synchronizing we don't want to advertise previous blocks
+    if (Globals.blockCount.get() < blockCount) {
+      log.debug(s"current blockchain height=$blockCount")
+      system.eventStream.publish(CurrentBlockCount(blockCount))
+      Globals.blockCount.set(blockCount)
+    }
+  }
+
+  val addressSubscriptions = collection.mutable.HashMap.empty[String, Set[ActorRef]]
+  val scriptHashSubscriptions = collection.mutable.HashMap.empty[BinaryData, Set[ActorRef]]
+  val headerSubscriptions = collection.mutable.HashSet.empty[ActorRef]
+
+  context.system.eventStream.publish(ElectrumDisconnected)
+  self ! Tcp.Connect(serverAddresses.head)
+
+  var reqId = 0L
+
+  def receive = disconnected
+
+  def disconnected: Receive = {
+    case c: Tcp.Connect =>
+      log.info(s"connecting to $c")
+      IO(Tcp) ! c
+
+    case Tcp.Connected(remote, _) =>
+      log.info(s"connected to $remote")
+      connectionFailures.clear()
+      val connection = sender()
+      connection ! Tcp.Register(self)
+      val request = version
+      send(connection, makeRequest(request, "" + reqId))
+      reqId = reqId + 1
+      context become waitingForVersion(connection, remote)
+
+    case AddStatusListener(actor) => statusListeners += actor
+
+    case Tcp.CommandFailed(Tcp.Connect(remoteAddress, _, _, _, _)) =>
+      val nextAddress = nextPeer()
+      log.warning(s"connection to $remoteAddress failed, trying $nextAddress")
+      connectionFailures.put(remoteAddress, connectionFailures.getOrElse(remoteAddress, 0L) + 1L)
+      val count = connectionFailures.getOrElse(nextAddress, 0L)
+      val delay = Math.min(Math.pow(2.0, count), 60.0) seconds;
+      context.system.scheduler.scheduleOnce(delay, self, Tcp.Connect(nextAddress))
+  }
+
+  def waitingForVersion(connection: ActorRef, remote: InetSocketAddress): Receive = {
+    case Tcp.Received(data) =>
+      val response = parseResponse(new String(data.toArray)).right.get
+      val serverVersion = parseJsonResponse(version, response)
+      log.debug(s"serverVersion=$serverVersion")
+      val request = HeaderSubscription(self)
+      send(connection, makeRequest(request, "" + reqId))
+      headerSubscriptions += self
+      log.debug("waiting for tip")
+      reqId = reqId + 1
+      context become waitingForTip(connection, remote: InetSocketAddress)
+
+    case AddStatusListener(actor) => statusListeners += actor
+  }
+
+  def waitingForTip(connection: ActorRef, remote: InetSocketAddress): Receive = {
+    case Tcp.Received(data) =>
+      val response = parseResponse(new String(data.toArray)).right.get
+      val header = parseHeader(response.result)
+      log.debug(s"connected, tip = ${header.block_hash} $header")
+      updateBlockCount(header.block_height)
+      statusListeners.map(_ ! ElectrumReady)
+      context.system.eventStream.publish(ElectrumConnected)
+      context become connected(connection, remote, header, "", Map.empty)
+
+    case AddStatusListener(actor) => statusListeners += actor
+  }
+
+  def connected(connection: ActorRef, remoteAddress: InetSocketAddress, tip: Header, buffer: String, requests: Map[String, (Request, ActorRef)]): Receive = {
+    case AddStatusListener(actor) =>
+      statusListeners += actor
+      actor ! ElectrumReady
+
+    case HeaderSubscription(actor) =>
+      headerSubscriptions += actor
+      actor ! HeaderSubscriptionResponse(tip)
+      context watch actor
+
+    case request: Request =>
+      val curReqId = "" + reqId
+      send(connection, makeRequest(request, curReqId))
+      request match {
+        case AddressSubscription(address, actor) =>
+          addressSubscriptions.update(address, addressSubscriptions.getOrElse(address, Set()) + actor)
+          context watch actor
+        case ScriptHashSubscription(scriptHash, actor) =>
+          scriptHashSubscriptions.update(scriptHash, scriptHashSubscriptions.getOrElse(scriptHash, Set()) + actor)
+          context watch actor
+        case _ => ()
+      }
+      reqId = reqId + 1
+      context become connected(connection, remoteAddress, tip, buffer, requests + (curReqId -> (request, sender())))
+
+    case Tcp.Received(data) =>
+      val buffer1 = buffer + new String(data.toArray)
+      val (jsons, buffer2) = buffer1.split(newline) match {
+        case chunks if buffer1.endsWith(newline) => (chunks, "")
+        case chunks => (chunks.dropRight(1), chunks.last)
+      }
+      jsons.map(parseResponse(_)).map(self ! _)
+      context become connected(connection, remoteAddress, tip, buffer2, requests)
+
+    case Right(json: JsonRPCResponse) =>
+      requests.get(json.id) match {
+        case Some((request, requestor)) =>
+          val response = parseJsonResponse(request, json)
+          log.debug(s"got response for reqId=${json.id} request=$request response=$response")
+          requestor ! response
+        case None =>
+          log.warning(s"could not find requestor for reqId=${json.id} response=$json")
+      }
+      context become connected(connection, remoteAddress, tip, buffer, requests - json.id)
+
+    case Left(response: HeaderSubscriptionResponse) => headerSubscriptions.map(_ ! response)
+
+    case Left(response: AddressSubscriptionResponse) => addressSubscriptions.get(response.address).map(listeners => listeners.map(_ ! response))
+
+    case Left(response: ScriptHashSubscriptionResponse) => scriptHashSubscriptions.get(response.scriptHash).map(listeners => listeners.map(_ ! response))
+
+    case HeaderSubscriptionResponse(newtip) =>
+      log.info(s"new tip $newtip")
+      updateBlockCount(newtip.block_height)
+      context become connected(connection, remoteAddress, newtip, buffer, requests)
+  }
+
+}
+
+object ElectrumClient {
+
+  def apply(addresses: java.util.List[InetSocketAddress]): ElectrumClient = {
+    import collection.JavaConversions._
+    new ElectrumClient(addresses)
+  }
+
+  /**
+    * Utility function to converts a publicKeyScript to electrum's scripthash
+    *
+    * @param publicKeyScript public key script
+    * @return the hash of the public key script, as used by ElectrumX's hash-based methods
+    */
+  def computeScriptHash(publicKeyScript: BinaryData): BinaryData = Crypto.sha256(publicKeyScript).reverse
+
+  // @formatter:off
+  sealed trait Request
+  sealed trait Response
+
+  case class ServerVersion(clientName: String, protocolVersion: String) extends Request
+  case class ServerVersionResponse(clientName: String, protocolVersion: String) extends Response
+
+  case class GetAddressHistory(address: String) extends Request
+  case class TransactionHistoryItem(height: Long, tx_hash: BinaryData)
+  case class GetAddressHistoryResponse(address: String, history: Seq[TransactionHistoryItem]) extends Response
+
+  case class GetScriptHashHistory(scriptHash: BinaryData) extends Request
+  case class GetScriptHashHistoryResponse(scriptHash: BinaryData, history: Seq[TransactionHistoryItem]) extends Response
+
+  case class AddressListUnspent(address: String) extends Request
+  case class UnspentItem(tx_hash: BinaryData, tx_pos: Int, value: Long, height: Long) {
+    lazy val outPoint = OutPoint(tx_hash.reverse, tx_pos)
+  }
+  case class AddressListUnspentResponse(address: String, unspents: Seq[UnspentItem]) extends Response
+
+  case class ScriptHashListUnspent(scriptHash: BinaryData) extends Request
+  case class ScriptHashListUnspentResponse(scriptHash: BinaryData, unspents: Seq[UnspentItem]) extends Response
+
+  case class BroadcastTransaction(tx: Transaction) extends Request
+  case class BroadcastTransactionResponse(tx: Transaction, error: Option[Error]) extends Response
+
+  case class GetTransaction(txid: BinaryData) extends Request
+  case class GetTransactionResponse(tx: Transaction) extends Response
+
+  case class GetMerkle(txid: BinaryData, height: Long) extends Request
+  case class GetMerkleResponse(txid: BinaryData, merkle: Seq[BinaryData], block_height: Long, pos: Int) extends Response {
+    lazy val root: BinaryData = {
+      @tailrec
+      def loop(pos: Int, hashes: Seq[BinaryData]): BinaryData = {
+        if (hashes.length == 1) hashes(0).reverse
+        else {
+          val h = if (pos % 2 == 1) Crypto.hash256(hashes(1) ++ hashes(0)) else Crypto.hash256(hashes(0) ++ hashes(1))
+          loop(pos / 2, h +: hashes.drop(2))
+        }
+      }
+      loop(pos, BinaryData(txid.reverse) +: merkle.map(b => BinaryData(b.reverse)))
+    }
+  }
+
+  case class AddressSubscription(address: String, actor: ActorRef) extends Request
+  case class AddressSubscriptionResponse(address: String, status: String) extends Response
+
+  case class ScriptHashSubscription(scriptHash: BinaryData, actor: ActorRef) extends Request
+  case class ScriptHashSubscriptionResponse(scriptHash: BinaryData, status: String) extends Response
+
+  case class HeaderSubscription(actor: ActorRef) extends Request
+  case class HeaderSubscriptionResponse(header: Header) extends Response
+
+  case class Header(block_height: Long, version: Long, prev_block_hash: BinaryData, merkle_root: BinaryData, timestamp: Long, bits: Long, nonce: Long) {
+    lazy val block_hash: BinaryData = {
+      val blockHeader = BlockHeader(version, prev_block_hash.reverse, merkle_root.reverse, timestamp, bits, nonce)
+      blockHeader.hash.reverse
+    }
+  }
+
+  object Header {
+    def makeHeader(height: Long, header: BlockHeader) = ElectrumClient.Header(0, header.version, header.hashPreviousBlock, header.hashMerkleRoot, header.time, header.bits, header.nonce)
+
+    val RegtestGenesisHeader = makeHeader(0, Block.RegtestGenesisBlock.header)
+    val TestnetGenesisHeader = makeHeader(0, Block.TestnetGenesisBlock.header)
+  }
+
+  case class TransactionHistory(history: Seq[TransactionHistoryItem]) extends Response
+
+  case class AddressStatus(address: String, status: String) extends Response
+
+  case class ServerError(request: Request, error: Error) extends Response
+  case class AddStatusListener(actor: ActorRef) extends Response
+
+  sealed trait ElectrumEvent
+  case object ElectrumConnected extends ElectrumEvent
+  case object ElectrumReady extends ElectrumEvent
+  case object ElectrumDisconnected extends ElectrumEvent
+
+  // @formatter:on
+
+  def parseResponse(input: String): Either[Response, JsonRPCResponse] = {
+    implicit val formats = DefaultFormats
+    val json = JsonMethods.parse(new String(input))
+    json \ "method" match {
+      case JString(method) =>
+        // this is a jsonrpc request, i.e. a subscription response
+        val JArray(params) = json \ "params"
+        Left(((method, params): @unchecked) match {
+          case ("blockchain.headers.subscribe", header :: Nil) => HeaderSubscriptionResponse(parseHeader(header))
+          case ("blockchain.address.subscribe", JString(address) :: JNull :: Nil) => AddressSubscriptionResponse(address, "")
+          case ("blockchain.address.subscribe", JString(address) :: JString(status) :: Nil) => AddressSubscriptionResponse(address, status)
+          case ("blockchain.scripthash.subscribe", JString(scriptHashHex) :: JNull :: Nil) => ScriptHashSubscriptionResponse(BinaryData(scriptHashHex), "")
+          case ("blockchain.scripthash.subscribe", JString(scriptHashHex) :: JString(status) :: Nil) => ScriptHashSubscriptionResponse(BinaryData(scriptHashHex), status)
+        })
+      case _ => Right(parseJsonRpcResponse(json))
+    }
+  }
+
+  def parseJsonRpcResponse(json: JValue): JsonRPCResponse = {
+    implicit val formats = DefaultFormats
+    val result = json \ "result"
+    val error = json \ "error" match {
+      case JNull => None
+      case JNothing => None
+      case other =>
+        val message = other \ "message" match {
+          case JString(value) => value
+          case _ => ""
+        }
+        val code = other \ " code" match {
+          case JInt(value) => value.intValue()
+          case JLong(value) => value.intValue()
+          case _ => 0
+        }
+        Some(Error(code, message))
+    }
+    val id = json \ "id" match {
+      case JString(value) => value
+      case JInt(value) => value.toString()
+      case JLong(value) => value.toString
+      case _ => ""
+    }
+    JsonRPCResponse(result, error, id)
+  }
+
+  def longField(jvalue: JValue, field: String): Long = (jvalue \ field: @unchecked) match {
+    case JLong(value) => value.longValue()
+    case JInt(value) => value.longValue()
+  }
+
+  def intField(jvalue: JValue, field: String): Int = (jvalue \ field: @unchecked) match {
+    case JLong(value) => value.intValue()
+    case JInt(value) => value.intValue()
+  }
+
+  def parseHeader(json: JValue): Header = {
+    val block_height = longField(json, "block_height")
+    val version = longField(json, "version")
+    val timestamp = longField(json, "timestamp")
+    val bits = longField(json, "bits")
+    val nonce = longField(json, "nonce")
+    val JString(prev_block_hash) = json \ "prev_block_hash"
+    val JString(merkle_root) = json \ "merkle_root"
+    Header(block_height, version, prev_block_hash, merkle_root, timestamp, bits, nonce)
+  }
+
+  def makeRequest(request: Request, reqId: String): JsonRPCRequest = request match {
+    case ServerVersion(clientName, protocolVersion) => JsonRPCRequest(id = reqId, method = "server.version", params = clientName :: protocolVersion :: Nil)
+    case GetAddressHistory(address) => JsonRPCRequest(id = reqId, method = "blockchain.address.get_history", params = address :: Nil)
+    case GetScriptHashHistory(scripthash) => JsonRPCRequest(id = reqId, method = "blockchain.scripthash.get_history", params = scripthash.toString() :: Nil)
+    case AddressListUnspent(address) => JsonRPCRequest(id = reqId, method = "blockchain.address.listunspent", params = address :: Nil)
+    case ScriptHashListUnspent(scripthash) => JsonRPCRequest(id = reqId, method = "blockchain.scripthash.listunspent", params = scripthash.toString() :: Nil)
+    case AddressSubscription(address, _) => JsonRPCRequest(id = reqId, method = "blockchain.address.subscribe", params = address :: Nil)
+    case ScriptHashSubscription(scriptHash, _) => JsonRPCRequest(id = reqId, method = "blockchain.scripthash.subscribe", params = scriptHash.toString() :: Nil)
+    case BroadcastTransaction(tx) => JsonRPCRequest(id = reqId, method = "blockchain.transaction.broadcast", params = Hex.toHexString(Transaction.write(tx)) :: Nil)
+    case GetTransaction(txid: BinaryData) => JsonRPCRequest(id = reqId, method = "blockchain.transaction.get", params = txid :: Nil)
+    case HeaderSubscription(_) => JsonRPCRequest(id = reqId, method = "blockchain.headers.subscribe", params = Nil)
+    case GetMerkle(txid, height) => JsonRPCRequest(id = reqId, method = "blockchain.transaction.get_merkle", params = txid :: height :: Nil)
+  }
+
+  def parseJsonResponse(request: Request, json: JsonRPCResponse): Response = {
+    implicit val formats = DefaultFormats
+    json.error match {
+      case Some(error) => (request: @unchecked) match {
+        case BroadcastTransaction(tx) => BroadcastTransactionResponse(tx, Some(error)) // for this request type, error are considered a "normal" response
+        case _ => ServerError(request, error)
+      }
+      case None => (request: @unchecked) match {
+        case s: ServerVersion =>
+          val JArray(jitems) = json.result
+          val JString(clientName) = jitems(0)
+          val JString(protocolVersion) = jitems(1)
+          ServerVersionResponse(clientName, protocolVersion)
+        case GetAddressHistory(address) =>
+          val JArray(jitems) = json.result
+          val items = jitems.map(jvalue => {
+            val JString(tx_hash) = jvalue \ "tx_hash"
+            val height = longField(jvalue, "height")
+            TransactionHistoryItem(height, tx_hash)
+          })
+          GetAddressHistoryResponse(address, items)
+        case GetScriptHashHistory(scripthash) =>
+          val JArray(jitems) = json.result
+          val items = jitems.map(jvalue => {
+            val JString(tx_hash) = jvalue \ "tx_hash"
+            val height = longField(jvalue, "height")
+            TransactionHistoryItem(height, tx_hash)
+          })
+          GetScriptHashHistoryResponse(scripthash, items)
+        case AddressListUnspent(address) =>
+          val JArray(jitems) = json.result
+          val items = jitems.map(jvalue => {
+            val JString(tx_hash) = jvalue \ "tx_hash"
+            val tx_pos = intField(jvalue, "tx_pos")
+            val height = longField(jvalue, "height")
+            val value = longField(jvalue, "value")
+            UnspentItem(tx_hash, tx_pos, value, height)
+          })
+          AddressListUnspentResponse(address, items)
+        case ScriptHashListUnspent(scripthash) =>
+          val JArray(jitems) = json.result
+          val items = jitems.map(jvalue => {
+            val JString(tx_hash) = jvalue \ "tx_hash"
+            val tx_pos = intField(jvalue, "tx_pos")
+            val height = longField(jvalue, "height")
+            val value = longField(jvalue, "value")
+            UnspentItem(tx_hash, tx_pos, value, height)
+          })
+          ScriptHashListUnspentResponse(scripthash, items)
+        case GetTransaction(_) =>
+          val JString(hex) = json.result
+          GetTransactionResponse(Transaction.read(hex))
+        case AddressSubscription(address, _) => json.result match {
+          case JString(status) => AddressSubscriptionResponse(address, status)
+          case _ => AddressSubscriptionResponse(address, "")
+        }
+        case ScriptHashSubscription(scriptHash, _) => json.result match {
+          case JString(status) => ScriptHashSubscriptionResponse(scriptHash, status)
+          case _ => ScriptHashSubscriptionResponse(scriptHash, "")
+        }
+        case BroadcastTransaction(tx) =>
+          val JString(txid) = json.result
+          require(BinaryData(txid) == tx.txid)
+          BroadcastTransactionResponse(tx, None)
+        case GetMerkle(txid, height) =>
+          val JArray(hashes) = json.result \ "merkle"
+          val leaves = hashes collect { case JString(value) => BinaryData(value) }
+          val blockHeight = longField(json.result, "block_height")
+          val JInt(pos) = json.result \ "pos"
+          GetMerkleResponse(txid, leaves, blockHeight, pos.toInt)
+      }
+    }
+  }
+
+  def readServerAddresses(stream: InputStream): Seq[InetSocketAddress] = try {
+    val JObject(values) = JsonMethods.parse(stream)
+    val addresses = values.map {
+      case (name, fields) =>
+        val JString(port) = fields \ "t"
+        new InetSocketAddress(name, port.toInt)
+    }
+    val randomized = Random.shuffle(addresses)
+    randomized
+  } finally {
+    stream.close()
+  }
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumEclairWallet.scala

@@ -0,0 +1,69 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import akka.actor.{ActorRef, ActorSystem}
+import akka.pattern.ask
+import fr.acinq.bitcoin.{Base58, Base58Check, BinaryData, OP_EQUAL, OP_HASH160, OP_PUSHDATA, Satoshi, Script, Transaction, TxOut}
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient.BroadcastTransaction
+import fr.acinq.eclair.blockchain.electrum.ElectrumWallet._
+import fr.acinq.eclair.blockchain.{EclairWallet, MakeFundingTxResponse}
+import grizzled.slf4j.Logging
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class ElectrumEclairWallet(val wallet: ActorRef)(implicit system: ActorSystem, ec: ExecutionContext, timeout: akka.util.Timeout) extends EclairWallet with Logging {
+
+  override def getBalance = (wallet ? GetBalance).mapTo[GetBalanceResponse].map(balance => balance.confirmed + balance.unconfirmed)
+
+  override def getFinalAddress = (wallet ? GetCurrentReceiveAddress).mapTo[GetCurrentReceiveAddressResponse].map(_.address)
+
+  override def makeFundingTx(pubkeyScript: BinaryData, amount: Satoshi, feeRatePerKw: Long) = {
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(amount, pubkeyScript) :: Nil, lockTime = 0)
+    (wallet ? CompleteTransaction(tx, feeRatePerKw)).mapTo[CompleteTransactionResponse].map(response => response match {
+      case CompleteTransactionResponse(tx1, None) => MakeFundingTxResponse(tx1, 0)
+      case CompleteTransactionResponse(_, Some(error)) => throw error
+    })
+  }
+
+  override def commit(tx: Transaction): Future[Boolean] =
+    (wallet ? BroadcastTransaction(tx)) flatMap {
+      case ElectrumClient.BroadcastTransactionResponse(tx, None) =>
+        //tx broadcast successfully: commit tx
+        wallet ? CommitTransaction(tx)
+      case ElectrumClient.BroadcastTransactionResponse(tx, Some(error)) if error.message.contains("transaction already in block chain") =>
+        // tx was already in the blockchain, that's weird but it is OK
+        wallet ? CommitTransaction(tx)
+      case ElectrumClient.BroadcastTransactionResponse(_, Some(error)) =>
+        //tx broadcast failed: cancel tx
+        logger.error(s"cannot broadcast tx ${tx.txid}: $error")
+        wallet ? CancelTransaction(tx)
+      case ElectrumClient.ServerError(ElectrumClient.BroadcastTransaction(tx), error) =>
+        //tx broadcast failed: cancel tx
+        logger.error(s"cannot broadcast tx ${tx.txid}: $error")
+        wallet ? CancelTransaction(tx)
+    } map {
+      case CommitTransactionResponse(_) => true
+      case CancelTransactionResponse(_) => false
+    }
+
+  def sendPayment(amount: Satoshi, address: String, feeRatePerKw: Long): Future[String] = {
+    val publicKeyScript = Base58Check.decode(address) match {
+      case (Base58.Prefix.PubkeyAddressTestnet, pubKeyHash) => Script.pay2pkh(pubKeyHash)
+      case (Base58.Prefix.ScriptAddressTestnet, scriptHash) => OP_HASH160 :: OP_PUSHDATA(scriptHash) :: OP_EQUAL :: Nil
+    }
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(amount, publicKeyScript) :: Nil, lockTime = 0)
+
+    (wallet ? CompleteTransaction(tx, feeRatePerKw))
+      .mapTo[CompleteTransactionResponse]
+      .flatMap {
+        case CompleteTransactionResponse(tx, None) => commit(tx).map {
+          case true => tx.txid.toString()
+          case false => throw new RuntimeException(s"could not commit tx=${Transaction.write(tx)}")
+        }
+        case CompleteTransactionResponse(_, Some(error)) => throw error
+      }
+  }
+
+  def getMnemonics: Future[Seq[String]] = (wallet ? GetMnemonicCode).mapTo[GetMnemonicCodeResponse].map(_.mnemonics)
+
+  override def rollback(tx: Transaction): Future[Boolean] = (wallet ? CancelTransaction(tx)).map(_ => true)
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWallet.scala

@@ -0,0 +1,693 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import java.io.File
+
+import akka.actor.{ActorRef, LoggingFSM, Props}
+import com.google.common.io.Files
+import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
+import fr.acinq.bitcoin.DeterministicWallet.{ExtendedPrivateKey, derivePrivateKey, hardened}
+import fr.acinq.bitcoin.{Base58, Base58Check, BinaryData, Block, Crypto, DeterministicWallet, MnemonicCode, OP_PUSHDATA, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptFlags, ScriptWitness, SigVersion, Transaction, TxIn, TxOut}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.Error
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient.{GetTransaction, GetTransactionResponse, TransactionHistoryItem, computeScriptHash}
+import fr.acinq.eclair.randomBytes
+import grizzled.slf4j.Logging
+
+import scala.annotation.tailrec
+import scala.util.{Failure, Success, Try}
+
+/**
+  * Simple electrum wallet
+  *
+  * Typical workflow:
+  *
+  * client ---- header update ----> wallet
+  * client ---- status update ----> wallet
+  * client <--- ask history   ----- wallet
+  * client ---- history       ----> wallet
+  * client <--- ask tx        ----- wallet
+  * client ---- tx            ----> wallet
+  *
+  * @param mnemonics
+  * @param client
+  * @param params
+  */
+class ElectrumWallet(mnemonics: Seq[String], client: ActorRef, params: ElectrumWallet.WalletParameters) extends LoggingFSM[ElectrumWallet.State, ElectrumWallet.Data] {
+
+  import ElectrumWallet._
+  import params._
+
+  val seed = MnemonicCode.toSeed(mnemonics, "")
+  val master = DeterministicWallet.generate(seed)
+
+  val accountMaster = accountKey(master)
+  val changeMaster = changeKey(master)
+
+  client ! ElectrumClient.AddStatusListener(self)
+
+  // disconnected --> waitingForTip --> running --
+  // ^                                            |
+  // |                                            |
+  //  --------------------------------------------
+
+  startWith(DISCONNECTED, {
+    val header = chainHash match {
+      case Block.RegtestGenesisBlock.hash => ElectrumClient.Header.RegtestGenesisHeader
+      case Block.TestnetGenesisBlock.hash => ElectrumClient.Header.TestnetGenesisHeader
+    }
+    val firstAccountKeys = (0 until params.swipeRange).map(i => derivePrivateKey(accountMaster, i)).toVector
+    val firstChangeKeys = (0 until params.swipeRange).map(i => derivePrivateKey(changeMaster, i)).toVector
+    val data = Data(params, header, firstAccountKeys, firstChangeKeys)
+    context.system.eventStream.publish(NewWalletReceiveAddress(data.currentReceiveAddress))
+    data
+  })
+
+  when(DISCONNECTED) {
+    case Event(ElectrumClient.ElectrumReady, data) =>
+      client ! ElectrumClient.HeaderSubscription(self)
+      goto(WAITING_FOR_TIP) using data
+  }
+
+  when(WAITING_FOR_TIP) {
+    case Event(ElectrumClient.HeaderSubscriptionResponse(header), data) =>
+      data.accountKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
+      data.changeKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
+      goto(RUNNING) using data.copy(tip = header)
+
+    case Event(ElectrumClient.ElectrumDisconnected, data) =>
+      log.info(s"wallet got disconnected")
+      goto(DISCONNECTED) using data
+  }
+
+  when(RUNNING) {
+    case Event(ElectrumClient.HeaderSubscriptionResponse(header), data) if data.tip == header => stay
+
+    case Event(ElectrumClient.HeaderSubscriptionResponse(header), data) =>
+      log.info(s"got new tip ${header.block_hash} at ${header.block_height}")
+      data.heights.collect {
+        case (txid, height) if height > 0 =>
+          val confirmations = computeDepth(header.block_height, height)
+          context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+      }
+      stay using data.copy(tip = header)
+
+    case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if data.status.get(scriptHash) == Some(status) => stay // we already have it
+
+    case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if !data.accountKeyMap.contains(scriptHash) && !data.changeKeyMap.contains(scriptHash) =>
+      log.warning(s"received status=$status for scriptHash=$scriptHash which does not match any of our keys")
+      stay
+
+    case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if status == "" =>
+      val data1 = data.copy(status = data.status + (scriptHash -> status)) // empty status, nothing to do
+      goto(stateName) using data1
+
+    case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) =>
+      val key = data.accountKeyMap.getOrElse(scriptHash, data.changeKeyMap(scriptHash))
+      val isChange = data.changeKeyMap.contains(scriptHash)
+      log.info(s"received status=$status for scriptHash=$scriptHash key=${segwitAddress(key)} isChange=$isChange")
+
+      // let's retrieve the tx history for this key
+      client ! ElectrumClient.GetScriptHashHistory(scriptHash)
+
+      val (newAccountKeys, newChangeKeys) = data.status.get(status) match {
+        case None =>
+          // first time this script hash is used, need to generate a new key
+          val newKey = if (isChange) derivePrivateKey(changeMaster, data.changeKeys.last.path.lastChildNumber + 1) else derivePrivateKey(accountMaster, data.accountKeys.last.path.lastChildNumber + 1)
+          val newScriptHash = computeScriptHashFromPublicKey(newKey.publicKey)
+          log.info(s"generated key with index=${newKey.path.lastChildNumber} scriptHash=$newScriptHash key=${segwitAddress(newKey)} isChange=$isChange")
+          // listens to changes for the newly generated key
+          client ! ElectrumClient.ScriptHashSubscription(newScriptHash, self)
+          if (isChange) (data.accountKeys, data.changeKeys :+ newKey) else (data.accountKeys :+ newKey, data.changeKeys)
+        case Some(_) => (data.accountKeys, data.changeKeys)
+      }
+
+      val data1 = data.copy(
+        accountKeys = newAccountKeys,
+        changeKeys = newChangeKeys,
+        status = data.status + (scriptHash -> status),
+        pendingHistoryRequests = data.pendingHistoryRequests + scriptHash)
+
+      goto(stateName) using data1 // goto instead of stay because we want to fire transitions
+
+    case Event(ElectrumClient.GetScriptHashHistoryResponse(scriptHash, history), data) =>
+      log.debug(s"scriptHash=$scriptHash has history=$history")
+      val (heights1, pendingTransactionRequests1) = history.foldLeft((data.heights, data.pendingTransactionRequests)) {
+        case ((heights, hashes), item) if !data.transactions.contains(item.tx_hash) && !data.pendingTransactionRequests.contains(item.tx_hash) =>
+          // we retrieve the tx if we don't have it and haven't yet requested it
+          client ! GetTransaction(item.tx_hash)
+          (heights + (item.tx_hash -> item.height), hashes + item.tx_hash)
+        case ((heights, hashes), item) =>
+          // otherwise we just update the height
+          (heights + (item.tx_hash -> item.height), hashes)
+      }
+
+      // we now have updated height for all our transactions,
+      heights1.collect {
+        case (txid, height) =>
+          val confirmations = if (height <= 0) 0 else computeDepth(data.tip.block_height, height)
+          (data.heights.get(txid), height) match {
+            case (None, height) if height <= 0 =>
+            // height=0 => unconfirmed, height=-1 => unconfirmed and one input is unconfirmed
+            case (None, height) if height > 0 =>
+              // first time we get a height for this tx: either it was just confirmed, or we restarted the wallet
+              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+            case (Some(previousHeight), height) if previousHeight != height =>
+              // there was a reorg
+              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+            case (Some(previousHeight), height) if previousHeight == height =>
+            // no reorg, nothing to do
+          }
+      }
+      val data1 = data.copy(heights = heights1, history = data.history + (scriptHash -> history), pendingHistoryRequests = data.pendingHistoryRequests - scriptHash, pendingTransactionRequests = pendingTransactionRequests1)
+      goto(stateName) using data1 // goto instead of stay because we want to fire transitions
+
+    case Event(GetTransactionResponse(tx), data) =>
+      log.debug(s"received transaction ${tx.txid}")
+      data.computeTransactionDelta(tx) match {
+        case Some((received, sent, fee_opt)) =>
+          log.info(s"successfully connected txid=${tx.txid}")
+          context.system.eventStream.publish(TransactionReceived(tx, data.computeTransactionDepth(tx.txid), received, sent, fee_opt))
+          // when we have successfully processed a new tx, we retry all pending txes to see if they can be added now
+          data.pendingTransactions.foreach(self ! GetTransactionResponse(_))
+          val data1 = data.copy(transactions = data.transactions + (tx.txid -> tx), pendingTransactionRequests = data.pendingTransactionRequests - tx.txid, pendingTransactions = Nil)
+          goto(stateName) using data1 // goto instead of stay because we want to fire transitions
+        case None =>
+          // missing parents
+          log.info(s"couldn't connect txid=${tx.txid}")
+          val data1 = data.copy(pendingTransactions = data.pendingTransactions :+ tx)
+          stay using data1
+      }
+
+    case Event(CompleteTransaction(tx, feeRatePerKw), data) =>
+      Try(data.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, allowSpendUnconfirmed)) match {
+        case Success((data1, tx1)) => stay using data1 replying CompleteTransactionResponse(tx1, None)
+        case Failure(t) => stay replying CompleteTransactionResponse(tx, Some(t))
+      }
+
+    case Event(CommitTransaction(tx), data) =>
+      log.info(s"committing txid=${tx.txid}")
+      val data1 = data.commitTransaction(tx)
+      // we use the initial state to compute the effect of the tx
+      // note: we know that computeTransactionDelta and the fee will be defined, because we built the tx ourselves so
+      // we know all the parents
+      val (received, sent, Some(fee)) = data.computeTransactionDelta(tx).get
+      // we notify here because the tx won't be downloaded again (it has been added to the state at commit)
+      context.system.eventStream.publish(TransactionReceived(tx, data1.computeTransactionDepth(tx.txid), received, sent, Some(fee)))
+      goto(stateName) using data1 replying CommitTransactionResponse(tx) // goto instead of stay because we want to fire transitions
+
+    case Event(CancelTransaction(tx), data) =>
+      log.info(s"cancelling txid=${tx.txid}")
+      stay using data.cancelTransaction(tx) replying CancelTransactionResponse(tx)
+
+    case Event(bc@ElectrumClient.BroadcastTransaction(tx), _) =>
+      log.info(s"broadcasting txid=${tx.txid}")
+      client forward bc
+      stay
+
+    case Event(ElectrumClient.ElectrumDisconnected, data) =>
+      log.info(s"wallet got disconnected")
+      goto(DISCONNECTED) using data
+  }
+
+  whenUnhandled {
+    case Event(GetMnemonicCode, _) => stay replying GetMnemonicCodeResponse(mnemonics)
+
+    case Event(GetCurrentReceiveAddress, data) => stay replying GetCurrentReceiveAddressResponse(data.currentReceiveAddress)
+
+    case Event(GetBalance, data) =>
+      val (confirmed, unconfirmed) = data.balance
+      stay replying GetBalanceResponse(confirmed, unconfirmed)
+
+    case Event(GetData, data) => stay replying GetDataResponse(data)
+
+    case Event(ElectrumClient.BroadcastTransaction(tx), _) => stay replying ElectrumClient.BroadcastTransactionResponse(tx, Some(Error(-1, "wallet is not connected")))
+  }
+
+  onTransition {
+    case _ -> _ if nextStateData.isReady(params.swipeRange) =>
+      val ready = nextStateData.readyMessage
+      log.info(s"wallet is ready with $ready")
+      context.system.eventStream.publish(ready)
+      context.system.eventStream.publish(NewWalletReceiveAddress(nextStateData.currentReceiveAddress))
+  }
+
+  initialize()
+
+}
+
+object ElectrumWallet {
+
+  // use 32 bytes seed, which will generate a 24 words mnemonic code
+  val SEED_BYTES_LENGTH = 32
+
+  def props(mnemonics: Seq[String], client: ActorRef, params: WalletParameters): Props = Props(new ElectrumWallet(mnemonics, client, params))
+
+  def props(file: File, client: ActorRef, params: WalletParameters): Props = {
+    val entropy: BinaryData = (file.exists(), file.canRead(), file.isFile) match {
+      case (true, true, true) => Files.toByteArray(file)
+      case (false, _, _) =>
+        val buffer = randomBytes(SEED_BYTES_LENGTH)
+        Files.write(buffer, file)
+        buffer
+      case _ => throw new IllegalArgumentException(s"cannot create wallet:$file exist but cannot read from")
+    }
+    val mnemonics = MnemonicCode.toMnemonics(entropy)
+    Props(new ElectrumWallet(mnemonics, client, params))
+  }
+
+  case class WalletParameters(chainHash: BinaryData, minimumFee: Satoshi = Satoshi(2000), dustLimit: Satoshi = Satoshi(546), swipeRange: Int = 10, allowSpendUnconfirmed: Boolean = true)
+
+  // @formatter:off
+  sealed trait State
+  case object DISCONNECTED extends State
+  case object WAITING_FOR_TIP extends State
+  case object RUNNING extends State
+
+  sealed trait Request
+  sealed trait Response
+
+  case object GetMnemonicCode extends RuntimeException
+  case class GetMnemonicCodeResponse(mnemonics: Seq[String]) extends Response
+
+  case object GetBalance extends Request
+  case class GetBalanceResponse(confirmed: Satoshi, unconfirmed: Satoshi) extends Response
+
+  case object GetCurrentReceiveAddress extends Request
+  case class GetCurrentReceiveAddressResponse(address: String) extends Response
+
+  case object GetData extends Request
+  case class GetDataResponse(state: Data) extends Response
+
+  case class CompleteTransaction(tx: Transaction, feeRatePerKw: Long) extends Request
+  case class CompleteTransactionResponse(tx: Transaction, error: Option[Throwable]) extends Response
+
+  case class CommitTransaction(tx: Transaction) extends Request
+  case class CommitTransactionResponse(tx: Transaction) extends Response
+
+  case class SendTransaction(tx: Transaction) extends Request
+  case class SendTransactionReponse(tx: Transaction) extends Response
+
+  case class CancelTransaction(tx: Transaction) extends Request
+  case class CancelTransactionResponse(tx: Transaction) extends Response
+
+  case object InsufficientFunds extends Response
+  case class AmountBelowDustLimit(dustLimit: Satoshi) extends Response
+
+  case class GetPrivateKey(address: String) extends Request
+  case class GetPrivateKeyResponse(address: String, key: Option[ExtendedPrivateKey]) extends Response
+
+
+  sealed trait WalletEvent
+  /**
+    *
+    * @param tx
+    * @param depth
+    * @param received
+    * @param sent
+    * @param feeOpt is set only when we know it (i.e. for outgoing transactions)
+    */
+  case class TransactionReceived(tx: Transaction, depth: Long, received: Satoshi, sent: Satoshi, feeOpt: Option[Satoshi]) extends WalletEvent
+  case class TransactionConfidenceChanged(txid: BinaryData, depth: Long) extends WalletEvent
+  case class NewWalletReceiveAddress(address: String) extends WalletEvent
+  case class WalletReady(confirmedBalance: Satoshi, unconfirmedBalance: Satoshi, height: Long) extends WalletEvent
+  // @formatter:on
+
+  /**
+    *
+    * @param key public key
+    * @return the address of the p2sh-of-p2wpkh script for this key
+    */
+  def segwitAddress(key: PublicKey): String = {
+    val script = Script.pay2wpkh(key)
+    val hash = Crypto.hash160(Script.write(script))
+    Base58Check.encode(Base58.Prefix.ScriptAddressTestnet, hash)
+  }
+
+  def segwitAddress(key: ExtendedPrivateKey): String = segwitAddress(key.publicKey)
+
+  def segwitAddress(key: PrivateKey): String = segwitAddress(key.publicKey)
+
+  /**
+    *
+    * @param key public key
+    * @return a p2sh-of-p2wpkh script for this key
+    */
+  def computePublicKeyScript(key: PublicKey) = Script.pay2sh(Script.pay2wpkh(key))
+
+  /**
+    *
+    * @param key public key
+    * @return the hash of the public key script for this key, as used by ElectrumX's hash-based methods
+    */
+  def computeScriptHashFromPublicKey(key: PublicKey): BinaryData = Crypto.sha256(Script.write(computePublicKeyScript(key))).reverse
+
+  /**
+    * use BIP49 (and not BIP44) since we use p2sh-of-p2wpkh
+    *
+    * @param master master key
+    * @return the BIP49 account key for this master key: m/49'/1'/0'/0
+    */
+  def accountKey(master: ExtendedPrivateKey) = DeterministicWallet.derivePrivateKey(master, hardened(49) :: hardened(1) :: hardened(0) :: 0L :: Nil)
+
+  /**
+    * use BIP49 (and not BIP44) since we use p2sh-of-p2wpkh
+    *
+    * @param master master key
+    * @return the BIP49 change key for this master key: m/49'/1'/0'/1
+    */
+  def changeKey(master: ExtendedPrivateKey) = DeterministicWallet.derivePrivateKey(master, hardened(49) :: hardened(1) :: hardened(0) :: 1L :: Nil)
+
+  def totalAmount(utxos: Seq[Utxo]): Satoshi = Satoshi(utxos.map(_.item.value).sum)
+
+  def totalAmount(utxos: Set[Utxo]): Satoshi = totalAmount(utxos.toSeq)
+
+  /**
+    *
+    * @param weight       transaction weight
+    * @param feeRatePerKw fee rate
+    * @return the fee for this tx weight
+    */
+  def computeFee(weight: Int, feeRatePerKw: Long): Satoshi = Satoshi((weight * feeRatePerKw) / 1000)
+
+  /**
+    *
+    * @param txIn transaction input
+    * @return Some(pubkey) if this tx input spends a p2sh-of-p2wpkh(pub), None otherwise
+    */
+  def extractPubKeySpentFrom(txIn: TxIn): Option[PublicKey] = {
+    Try {
+      // we're looking for tx that spend a pay2sh-of-p2wkph output
+      require(txIn.witness.stack.size == 2)
+      val sig = txIn.witness.stack(0)
+      val pub = txIn.witness.stack(1)
+      val OP_PUSHDATA(script, _) :: Nil = Script.parse(txIn.signatureScript)
+      val publicKey = PublicKey(pub)
+      if (Script.write(Script.pay2wpkh(publicKey)) == script) {
+        Some(publicKey)
+      } else None
+    } getOrElse None
+  }
+
+  def computeDepth(currentHeight: Long, txHeight: Long): Long = currentHeight - txHeight + 1
+
+  case class Utxo(key: ExtendedPrivateKey, item: ElectrumClient.UnspentItem) {
+    def outPoint: OutPoint = item.outPoint
+  }
+
+  /**
+    * Wallet state, which stores data returned by EletrumX servers.
+    * Most items are indexed by script hash (i.e. by pubkey script sha256 hash).
+    * Height follow ElectrumX's conventions:
+    * - h > 0 means that the tx was confirmed at block #h
+    * - 0 means unconfirmed, but all input are confirmed
+    * < 0 means unconfirmed, and sonme inputs are unconfirmed as well
+    *
+    * @param tip          current blockchain tip
+    * @param accountKeys  account keys
+    * @param changeKeys   change keys
+    * @param status       script hash -> status; "" means that the script hash has not been used
+    *                     yet
+    * @param transactions wallet transactions
+    * @param heights      transactions heights
+    * @param history      script hash -> history
+    * @param locks        transactions which lock some of our utxos.
+    */
+  case class Data(tip: ElectrumClient.Header,
+                  accountKeys: Vector[ExtendedPrivateKey],
+                  changeKeys: Vector[ExtendedPrivateKey],
+                  status: Map[BinaryData, String],
+                  transactions: Map[BinaryData, Transaction],
+                  heights: Map[BinaryData, Long],
+                  history: Map[BinaryData, Seq[ElectrumClient.TransactionHistoryItem]],
+                  locks: Set[Transaction],
+                  pendingHistoryRequests: Set[BinaryData],
+                  pendingTransactionRequests: Set[BinaryData],
+                  pendingTransactions: Seq[Transaction]) extends Logging {
+    lazy val accountKeyMap = accountKeys.map(key => computeScriptHashFromPublicKey(key.publicKey) -> key).toMap
+
+    lazy val changeKeyMap = changeKeys.map(key => computeScriptHashFromPublicKey(key.publicKey) -> key).toMap
+
+    lazy val firstUnusedAccountKeys = accountKeys.find(key => status.get(computeScriptHashFromPublicKey(key.publicKey)) == Some(""))
+
+    lazy val firstUnusedChangeKeys = changeKeys.find(key => status.get(computeScriptHashFromPublicKey(key.publicKey)) == Some(""))
+
+    lazy val publicScriptMap = (accountKeys ++ changeKeys).map(key => Script.write(computePublicKeyScript(key.publicKey)) -> key).toMap
+
+    lazy val utxos = history.keys.toSeq.map(scriptHash => getUtxos(scriptHash)).flatten
+
+    /**
+      * The wallet is ready if all current keys have an empty status, and we don't have
+      * any history/tx request pending
+      * NB: swipeRange * 2 because we have account keys and change keys
+      */
+    def isReady(swipeRange: Int) = status.filter(_._2 == "").size >= swipeRange * 2 && pendingHistoryRequests.isEmpty && pendingTransactionRequests.isEmpty
+
+    def readyMessage: WalletReady = {
+      val (confirmed, unconfirmed) = balance
+      WalletReady(confirmed, unconfirmed, tip.block_height)
+    }
+
+    /**
+      *
+      * @return the current receive key. In most cases it will be a key that has not
+      *         been used yet but it may be possible that we are still looking for
+      *         unused keys and none is available yet. In this case we will return
+      *         the latest account key.
+      */
+    def currentReceiveKey = firstUnusedAccountKeys.headOption.getOrElse {
+      // bad luck we are still looking for unused keys
+      // use the first account key
+      accountKeys.head
+    }
+
+    def currentReceiveAddress = segwitAddress(currentReceiveKey)
+
+    /**
+      *
+      * @return the current change key. In most cases it will be a key that has not
+      *         been used yet but it may be possible that we are still looking for
+      *         unused keys and none is available yet. In this case we will return
+      *         the latest change key.
+      */
+    def currentChangeKey = firstUnusedChangeKeys.headOption.getOrElse {
+      // bad luck we are still looking for unused keys
+      // use the first account key
+      changeKeys.head
+    }
+
+    def currentChangeAddress = segwitAddress(currentChangeKey)
+
+    def isMine(txIn: TxIn): Boolean = extractPubKeySpentFrom(txIn).exists(pub => publicScriptMap.contains(Script.write(computePublicKeyScript(pub))))
+
+    def isSpend(txIn: TxIn, publicKey: PublicKey): Boolean = extractPubKeySpentFrom(txIn).contains(publicKey)
+
+    /**
+      *
+      * @param txIn
+      * @param scriptHash
+      * @return true if txIn spends from an address that matches scriptHash
+      */
+    def isSpend(txIn: TxIn, scriptHash: BinaryData): Boolean = extractPubKeySpentFrom(txIn).exists(pub => computeScriptHashFromPublicKey(pub) == scriptHash)
+
+    def isReceive(txOut: TxOut, scriptHash: BinaryData): Boolean = publicScriptMap.get(txOut.publicKeyScript).exists(key => computeScriptHashFromPublicKey(key.publicKey) == scriptHash)
+
+    def isMine(txOut: TxOut): Boolean = publicScriptMap.contains(txOut.publicKeyScript)
+
+    def computeTransactionDepth(txid: BinaryData): Long = heights.get(txid).map(height => if (height > 0) computeDepth(tip.block_height, height) else 0).getOrElse(0)
+
+    /**
+      *
+      * @param scriptHash script hash
+      * @return the list of UTXOs for this script hash (including unconfirmed UTXOs)
+      */
+    def getUtxos(scriptHash: BinaryData) = {
+      history.get(scriptHash) match {
+        case None => Seq()
+        case Some(items) if items.isEmpty => Seq()
+        case Some(items) =>
+          // this is the private key for this script hash
+          val key = accountKeyMap.getOrElse(scriptHash, changeKeyMap(scriptHash))
+
+          // find all transactions that send to or receive from this script hash
+          // we use collect because we may not yet have received all transactions in the history
+          val txs = items collect { case item if transactions.contains(item.tx_hash) => transactions(item.tx_hash) }
+
+          // find all tx outputs that send to our script hash
+          val unspents = items collect { case item if transactions.contains(item.tx_hash) =>
+            val tx = transactions(item.tx_hash)
+            val outputs = tx.txOut.zipWithIndex.filter { case (txOut, index) => isReceive(txOut, scriptHash) }
+            outputs.map { case (txOut, index) => Utxo(key, ElectrumClient.UnspentItem(item.tx_hash, index, txOut.amount.toLong, item.height)) }
+          } flatten
+
+          // and remove the outputs that are being spent. this is needed because we may have unconfirmed UTXOs
+          // that are spend by unconfirmed transactions
+          unspents.filterNot(utxo => txs.exists(tx => tx.txIn.exists(_.outPoint == utxo.outPoint)))
+      }
+    }
+
+
+    /**
+      *
+      * @param scriptHash script hash
+      * @return the (confirmed, unconfirmed) balance for this script hash. This balance may not
+      *         be up-to-date if we have not received all data we've asked for yet.
+      */
+    def balance(scriptHash: BinaryData): (Satoshi, Satoshi) = {
+      history.get(scriptHash) match {
+        case None => (Satoshi(0), Satoshi(0))
+
+        case Some(items) if items.isEmpty => (Satoshi(0), Satoshi(0))
+
+        case Some(items) =>
+          val (confirmedItems, unconfirmedItems) = items.partition(_.height > 0)
+          val confirmedTxs = confirmedItems.collect { case item if transactions.contains(item.tx_hash) => transactions(item.tx_hash) }
+          val unconfirmedTxs = unconfirmedItems.collect { case item if transactions.contains(item.tx_hash) => transactions(item.tx_hash) }
+          if (confirmedTxs.size + unconfirmedTxs.size < confirmedItems.size + unconfirmedItems.size) logger.warn(s"we have not received all transactions yet, balance will not be up to date")
+
+          def findOurSpentOutputs(txs: Seq[Transaction]): Seq[TxOut] = {
+            val inputs = txs.map(_.txIn).flatten.filter(txIn => isSpend(txIn, scriptHash))
+            val spentOutputs = inputs.map(_.outPoint).map(outPoint => transactions.get(outPoint.txid).map(_.txOut(outPoint.index.toInt))).flatten
+            spentOutputs
+          }
+
+          val confirmedSpents = findOurSpentOutputs(confirmedTxs)
+          val confirmedReceived = confirmedTxs.map(_.txOut).flatten.filter(txOut => isReceive(txOut, scriptHash))
+
+          val unconfirmedSpents = findOurSpentOutputs(unconfirmedTxs)
+          val unconfirmedReceived = unconfirmedTxs.map(_.txOut).flatten.filter(txOut => isReceive(txOut, scriptHash))
+
+          val confirmedBalance = confirmedReceived.map(_.amount).sum - confirmedSpents.map(_.amount).sum
+          val unconfirmedBalance = unconfirmedReceived.map(_.amount).sum - unconfirmedSpents.map(_.amount).sum
+
+          (confirmedBalance, unconfirmedBalance)
+      }
+    }
+
+    /**
+      *
+      * @return the (confirmed, unconfirmed) balance for this wallet. This balance may not
+      *         be up-to-date if we have not received all data we've asked for yet.
+      */
+    lazy val balance: (Satoshi, Satoshi) = {
+      (accountKeyMap.keys ++ changeKeyMap.keys).map(scriptHash => balance(scriptHash)).foldLeft((Satoshi(0), Satoshi(0))) {
+        case ((confirmed, unconfirmed), (confirmed1, unconfirmed1)) => (confirmed + confirmed1, unconfirmed + unconfirmed1)
+      }
+    }
+
+    /**
+      * Computes the effect of this transaction on the wallet
+      *
+      * @param tx input transaction
+      * @return an option:
+      *         - Some(received, sent, fee) where sent if what the tx spends from us, received is what the tx sends to us,
+      *         and fee is the fee for the tx) tuple where sent if what the tx spends from us, and received is what the tx sends to us
+      *         - None if we are missing one or more parent txs
+      */
+    def computeTransactionDelta(tx: Transaction): Option[(Satoshi, Satoshi, Option[Satoshi])] = {
+      val ourInputs = tx.txIn.filter(isMine)
+      // we need to make sure that for all inputs spending an output we control, we already  have the parent tx
+      // (otherwise we can't estimate our balance)
+      val missingParent = ourInputs.exists(txIn => !transactions.contains(txIn.outPoint.txid))
+      if (missingParent) {
+        None
+      } else {
+        val sent = ourInputs.map(txIn => transactions(txIn.outPoint.txid).txOut(txIn.outPoint.index.toInt)).map(_.amount).sum
+        val received = tx.txOut.filter(isMine).map(_.amount).sum
+        // if all the inputs were ours, we can compute the fee, otherwise we can't
+        val fee_opt = if (ourInputs.size == tx.txIn.size) Some(sent - tx.txOut.map(_.amount).sum) else None
+        Some((received, sent, fee_opt))
+      }
+    }
+
+    /**
+      *
+      * @param tx           input tx that has no inputs
+      * @param feeRatePerKw fee rate per kiloweight
+      * @param minimumFee   minimum fee
+      * @param dustLimit    dust limit
+      * @return a (state, tx) tuple where state has been updated and tx is a complete,
+      *         fully signed transaction that can be broadcast.
+      *         our utxos spent by this tx are locked and won't be available for spending
+      *         until the tx has been cancelled. If the tx is committed, they will be removed
+      */
+    def completeTransaction(tx: Transaction, feeRatePerKw: Long, minimumFee: Satoshi, dustLimit: Satoshi, allowSpendUnconfirmed: Boolean): (Data, Transaction) = {
+      require(tx.txIn.isEmpty, "cannot complete a tx that already has inputs")
+      require(feeRatePerKw >= 0, "fee rate cannot be negative")
+      val amount = tx.txOut.map(_.amount).sum
+      require(amount > dustLimit, "amount to send is below dust limit")
+      val fee = {
+        val estimatedFee = computeFee(700, feeRatePerKw)
+        if (estimatedFee < minimumFee) minimumFee else estimatedFee
+      }
+
+      @tailrec
+      def select(chooseFrom: Seq[Utxo], selected: Set[Utxo]): Set[Utxo] = {
+        if (totalAmount(selected) >= amount + fee) selected
+        else if (chooseFrom.isEmpty) Set()
+        else select(chooseFrom.tail, selected + chooseFrom.head)
+      }
+
+      // select utxos that are not locked by pending txs
+      val lockedOutputs = locks.map(_.txIn.map(_.outPoint)).flatten
+      val unlocked = utxos.filterNot(utxo => lockedOutputs.contains(utxo.outPoint))
+      val unlocked1 = if (allowSpendUnconfirmed) unlocked else unlocked.filter(_.item.height > 0)
+      val selected = select(unlocked1, Set()).toSeq
+      require(totalAmount(selected) >= amount + fee, "insufficient funds")
+
+      // add inputs
+      var tx1 = tx.copy(txIn = selected.map(utxo => TxIn(utxo.outPoint, Nil, TxIn.SEQUENCE_FINAL)))
+
+      // add change output
+      val change = totalAmount(selected) - amount - fee
+      if (change >= dustLimit) tx1 = tx1.addOutput(TxOut(change, computePublicKeyScript(currentChangeKey.publicKey)))
+
+      // sign
+      for (i <- 0 until tx1.txIn.size) {
+        val key = selected(i).key
+        val sig = Transaction.signInput(tx1, i, Script.pay2pkh(key.publicKey), SIGHASH_ALL, Satoshi(selected(i).item.value), SigVersion.SIGVERSION_WITNESS_V0, key.privateKey)
+        tx1 = tx1.updateWitness(i, ScriptWitness(sig :: key.publicKey.toBin :: Nil)).updateSigScript(i, OP_PUSHDATA(Script.write(Script.pay2wpkh(key.publicKey))) :: Nil)
+      }
+      Transaction.correctlySpends(tx1, selected.map(utxo => utxo.outPoint -> TxOut(Satoshi(utxo.item.value), computePublicKeyScript(utxo.key.publicKey))).toMap, ScriptFlags.STANDARD_SCRIPT_VERIFY_FLAGS)
+
+      val data1 = this.copy(locks = this.locks + tx1)
+      (data1, tx1)
+    }
+
+    /**
+      * unlocks input locked by a pending tx. call this method if the tx will not be used after all
+      *
+      * @param tx pending transaction
+      * @return an updated state
+      */
+    def cancelTransaction(tx: Transaction): Data = this.copy(locks = this.locks - tx)
+
+    /**
+      * remove all our utxos spent by this tx. call this method if the tx was broadcast successfully
+      *
+      * @param tx pending transaction
+      * @return an updated state
+      */
+    def commitTransaction(tx: Transaction): Data = {
+      // HACK! since we base our utxos computation on the history as seen by the electrum server (so that it is
+      // reorg-proof out of the box), we need to update the history  right away if we want to be able to build chained
+      // unconfirmed transactions. A few seconds later electrum will notify us and the entry will be overwritten.
+      // Note that we need to take into account both inputs and outputs, because there may be change.
+      val history1 = (tx.txIn.filter(isMine).map(extractPubKeySpentFrom).flatten.map(computeScriptHashFromPublicKey) ++ tx.txOut.filter(isMine).map(_.publicKeyScript).map(computeScriptHash))
+        .foldLeft(this.history) {
+          case (history, scriptHash) =>
+            val entry = history.get(scriptHash) match {
+              case None => Seq(TransactionHistoryItem(0, tx.txid))
+              case Some(items) if items.map(_.tx_hash).contains(tx.txid) => items
+              case Some(items) => items :+ TransactionHistoryItem(0, tx.txid)
+            }
+            history + (scriptHash -> entry)
+        }
+      this.copy(locks = this.locks - tx, transactions = this.transactions + (tx.txid -> tx), heights = this.heights + (tx.txid -> 0L), history = history1)
+    }
+  }
+
+  object Data {
+    def apply(params: ElectrumWallet.WalletParameters, tip: ElectrumClient.Header, accountKeys: Vector[ExtendedPrivateKey], changeKeys: Vector[ExtendedPrivateKey]): Data
+    = Data(tip, accountKeys, changeKeys, Map(), Map(), Map(), Map(), Set(), Set(), Set(), Seq())
+  }
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWatcher.scala

@@ -0,0 +1,225 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import java.net.InetSocketAddress
+
+import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Props, Stash, Terminated}
+import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.bitcoin.{BinaryData, Satoshi, Script, Transaction, TxIn, TxOut}
+import fr.acinq.eclair.blockchain._
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient._
+import fr.acinq.eclair.channel.{BITCOIN_FUNDING_DEPTHOK, BITCOIN_FUNDING_SPENT, BITCOIN_PARENT_TX_CONFIRMED}
+import fr.acinq.eclair.transactions.Scripts
+import fr.acinq.eclair.{Globals, fromShortId}
+
+import scala.collection.SortedMap
+
+
+class ElectrumWatcher(client: ActorRef) extends Actor with Stash with ActorLogging {
+
+  client ! ElectrumClient.AddStatusListener(self)
+
+  override def unhandled(message: Any): Unit = message match {
+    case ParallelGetRequest(announcements) => sender ! ParallelGetResponse(announcements.map {
+      case c =>
+        log.info(s"blindly validating channel=$c")
+        val pubkeyScript = Script.write(Script.pay2wsh(Scripts.multiSig2of2(PublicKey(c.bitcoinKey1), PublicKey(c.bitcoinKey2))))
+        val (_, _, outputIndex) = fromShortId(c.shortChannelId)
+        val fakeFundingTx = Transaction(
+          version = 2,
+          txIn = Seq.empty[TxIn],
+          txOut = List.fill(outputIndex + 1)(TxOut(Satoshi(0), pubkeyScript)), // quick and dirty way to be sure that the outputIndex'th output is of the expected format
+          lockTime = 0)
+        IndividualResult(c, Some(fakeFundingTx), true)
+    })
+    case _ => log.warning(s"unhandled message $message")
+  }
+
+  def receive = disconnected(Set.empty, Nil, SortedMap.empty)
+
+  def disconnected(watches: Set[Watch], publishQueue: Seq[PublishAsap], block2tx: SortedMap[Long, Seq[Transaction]]): Receive = {
+    case ElectrumClient.ElectrumReady =>
+      client ! ElectrumClient.HeaderSubscription(self)
+    case ElectrumClient.HeaderSubscriptionResponse(header) =>
+      watches.map(self ! _)
+      publishQueue.map(self ! _)
+      context become running(header, Set(), Map(), block2tx, Nil)
+    case watch: Watch => context become disconnected(watches + watch, publishQueue, block2tx)
+    case publish: PublishAsap => context become disconnected(watches, publishQueue :+ publish, block2tx)
+  }
+
+  def running(tip: ElectrumClient.Header, watches: Set[Watch], scriptHashStatus: Map[BinaryData, String], block2tx: SortedMap[Long, Seq[Transaction]], sent: Seq[Transaction]): Receive = {
+    case ElectrumClient.HeaderSubscriptionResponse(newtip) if tip == newtip => ()
+
+    case ElectrumClient.HeaderSubscriptionResponse(newtip) =>
+      log.info(s"new tip: ${newtip.block_hash} $newtip")
+      watches collect {
+        case watch: WatchConfirmed =>
+          val scriptHash = computeScriptHash(watch.publicKeyScript)
+          client ! ElectrumClient.GetScriptHashHistory(scriptHash)
+      }
+      val toPublish = block2tx.filterKeys(_ <= newtip.block_height)
+      toPublish.values.flatten.foreach(tx => self ! PublishAsap(tx))
+      context become running(newtip, watches, scriptHashStatus, block2tx -- toPublish.keys, sent)
+
+    case watch: Watch if watches.contains(watch) => ()
+
+    case watch@WatchSpent(_, txid, outputIndex, publicKeyScript, _) =>
+      val scriptHash = computeScriptHash(publicKeyScript)
+      log.info(s"added watch-spent on output=$txid:$outputIndex scriptHash=$scriptHash")
+      client ! ElectrumClient.ScriptHashSubscription(scriptHash, self)
+      context.watch(watch.channel)
+      context become running(tip, watches + watch, scriptHashStatus, block2tx, sent)
+
+    case watch@WatchSpentBasic(_, txid, outputIndex, publicKeyScript, _) =>
+      val scriptHash = computeScriptHash(publicKeyScript)
+      log.info(s"added watch-spent-basic on output=$txid:$outputIndex scriptHash=$scriptHash")
+      client ! ElectrumClient.ScriptHashSubscription(scriptHash, self)
+      context.watch(watch.channel)
+      context become running(tip, watches + watch, scriptHashStatus, block2tx, sent)
+
+    case watch@WatchConfirmed(_, txid, publicKeyScript, _, _) =>
+      val scriptHash = computeScriptHash(publicKeyScript)
+      log.info(s"added watch-confirmed on txid=$txid scriptHash=$scriptHash")
+      client ! ElectrumClient.GetScriptHashHistory(scriptHash)
+      context.watch(watch.channel)
+      context become running(tip, watches + watch, scriptHashStatus, block2tx, sent)
+
+    case Terminated(actor) =>
+      val watches1 = watches.filterNot(_.channel == actor)
+      context become running(tip, watches1, scriptHashStatus, block2tx, sent)
+
+    case ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status) =>
+      scriptHashStatus.get(scriptHash) match {
+        case Some(s) if s == status => log.debug(s"already have status=$status for scriptHash=$scriptHash")
+        case _ if status.isEmpty => log.info(s"empty status for scriptHash=$scriptHash")
+        case _ =>
+          log.info(s"new status=$status for scriptHash=$scriptHash")
+          client ! ElectrumClient.GetScriptHashHistory(scriptHash)
+      }
+      context become running(tip, watches, scriptHashStatus + (scriptHash -> status), block2tx, sent)
+
+    case ElectrumClient.GetScriptHashHistoryResponse(_, history) =>
+      // this is for WatchSpent/WatchSpentBasic
+      history.filter(_.height >= 0).map(item => client ! ElectrumClient.GetTransaction(item.tx_hash))
+      // this is for WatchConfirmed
+      history.collect {
+        case ElectrumClient.TransactionHistoryItem(height, tx_hash) if height > 0 => watches.collect {
+          case WatchConfirmed(_, txid, _, minDepth, _) if txid == tx_hash =>
+            val confirmations = tip.block_height - height + 1
+            log.info(s"txid=$txid was confirmed at height=$height and now has confirmations=$confirmations (currentHeight=${tip.block_height})")
+            if (confirmations >= minDepth) {
+              // we need to get the tx position in the block
+              client ! GetMerkle(tx_hash, height)
+            }
+        }
+      }
+
+    case ElectrumClient.GetMerkleResponse(tx_hash, _, height, pos) =>
+      val confirmations = tip.block_height - height + 1
+      val triggered = watches.collect {
+        case w@WatchConfirmed(channel, txid, _, minDepth, event) if txid == tx_hash && confirmations >= minDepth =>
+          log.info(s"txid=$txid had confirmations=$confirmations in block=$height pos=$pos")
+          channel ! WatchEventConfirmed(event, height.toInt, pos)
+          w
+      }
+      context become running(tip, watches -- triggered, scriptHashStatus, block2tx, sent)
+
+    case ElectrumClient.GetTransactionResponse(spendingTx) =>
+      val triggered = spendingTx.txIn.map(_.outPoint).flatMap(outPoint => watches.collect {
+        case WatchSpent(channel, txid, pos, _, event) if txid == outPoint.txid && pos == outPoint.index.toInt =>
+          log.info(s"output $txid:$pos spent by transaction ${spendingTx.txid}")
+          channel ! WatchEventSpent(event, spendingTx)
+          // NB: WatchSpent are permanent because we need to detect multiple spending of the funding tx
+          // They are never cleaned up but it is not a big deal for now (1 channel == 1 watch)
+          None
+        case w@WatchSpentBasic(channel, txid, pos, _, event) if txid == outPoint.txid && pos == outPoint.index.toInt =>
+          log.info(s"output $txid:$pos spent by transaction ${spendingTx.txid}")
+          channel ! WatchEventSpentBasic(event)
+          Some(w)
+      }).flatten
+      context become running(tip, watches -- triggered, scriptHashStatus, block2tx, sent)
+
+    case PublishAsap(tx) =>
+      val blockCount = Globals.blockCount.get()
+      val cltvTimeout = Scripts.cltvTimeout(tx)
+      val csvTimeout = Scripts.csvTimeout(tx)
+      if (csvTimeout > 0) {
+        require(tx.txIn.size == 1, s"watcher only supports tx with 1 input, this tx has ${tx.txIn.size} inputs")
+        val parentTxid = tx.txIn(0).outPoint.txid
+        log.info(s"txid=${tx.txid} has a relative timeout of $csvTimeout blocks, watching parenttxid=$parentTxid tx=${Transaction.write(tx)}")
+        val parentPublicKeyScript = WatchConfirmed.extractPublicKeyScript(tx.txIn.head.witness)
+        self ! WatchConfirmed(self, parentTxid, parentPublicKeyScript, minDepth = 1, BITCOIN_PARENT_TX_CONFIRMED(tx))
+      } else if (cltvTimeout > blockCount) {
+        log.info(s"delaying publication of txid=${tx.txid} until block=$cltvTimeout (curblock=$blockCount)")
+        val block2tx1 = block2tx.updated(cltvTimeout, block2tx.getOrElse(cltvTimeout, Seq.empty[Transaction]) :+ tx)
+        context become running(tip, watches, scriptHashStatus, block2tx1, sent)
+      } else {
+        log.info(s"publishing tx=${Transaction.write(tx)}")
+        client ! BroadcastTransaction(tx)
+        context become running(tip, watches, scriptHashStatus, block2tx, sent :+ tx)
+      }
+
+    case WatchEventConfirmed(BITCOIN_PARENT_TX_CONFIRMED(tx), blockHeight, _) =>
+      log.info(s"parent tx of txid=${tx.txid} has been confirmed")
+      val blockCount = Globals.blockCount.get()
+      val csvTimeout = Scripts.csvTimeout(tx)
+      val absTimeout = blockHeight + csvTimeout
+      if (absTimeout > blockCount) {
+        log.info(s"delaying publication of txid=${tx.txid} until block=$absTimeout (curblock=$blockCount)")
+        val block2tx1 = block2tx.updated(absTimeout, block2tx.getOrElse(absTimeout, Seq.empty[Transaction]) :+ tx)
+        context become running(tip, watches, scriptHashStatus, block2tx1, sent)
+      } else {
+        log.info(s"publishing tx=${Transaction.write(tx)}")
+        client ! BroadcastTransaction(tx)
+        context become running(tip, watches, scriptHashStatus, block2tx, sent :+ tx)
+      }
+
+    case ElectrumClient.BroadcastTransactionResponse(tx, error_opt) =>
+      error_opt match {
+        case None => log.info(s"broadcast succeeded for txid=${tx.txid} tx=${Transaction.write(tx)}")
+        case Some(error) if error.message.contains("transaction already in block chain") => log.info(s"broadcast ignored for txid=${tx.txid} tx=${Transaction.write(tx)} (tx was already in blockchain)")
+        case Some(error) => log.error(s"broadcast failed for txid=${tx.txid} tx=${Transaction.write(tx)} with error=$error")
+      }
+      context become running(tip, watches, scriptHashStatus, block2tx, sent diff Seq(tx))
+
+    case ElectrumClient.ElectrumDisconnected =>
+      // we remember watches and keep track of tx that have not yet been published
+      // we also re-send the txes that we previsouly sent but hadn't yet received the confirmation
+      context become disconnected(watches, sent.map(PublishAsap(_)), block2tx)
+  }
+
+}
+
+object ElectrumWatcher extends App {
+
+  val system = ActorSystem()
+
+  class Root extends Actor with ActorLogging {
+    val serverAddresses = Seq(new InetSocketAddress("localhost", 51000), new InetSocketAddress("localhost", 51001))
+    val client = context.actorOf(Props(new ElectrumClient(serverAddresses)), "client")
+    client ! ElectrumClient.AddStatusListener(self)
+
+    override def unhandled(message: Any): Unit = {
+      super.unhandled(message)
+      log.warning(s"unhandled message $message")
+    }
+
+    def receive = {
+      case ElectrumClient.ElectrumReady =>
+        log.info(s"starting watcher")
+        context become running(context.actorOf(Props(new ElectrumWatcher(client)), "watcher"))
+    }
+
+    def running(watcher: ActorRef): Receive = {
+      case watch: Watch => watcher forward watch
+    }
+  }
+
+  val root = system.actorOf(Props[Root], "root")
+  val scanner = new java.util.Scanner(System.in)
+  while (true) {
+    val tx = Transaction.read(scanner.nextLine())
+    root ! WatchSpent(root, tx.txid, 0, tx.txOut(0).publicKeyScript, BITCOIN_FUNDING_SPENT)
+    root ! WatchConfirmed(root, tx.txid, tx.txOut(0).publicKeyScript, 4L, BITCOIN_FUNDING_DEPTHOK)
+  }
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/BitcoinCoreFeeProvider.scala

@@ -1,6 +1,7 @@
 package fr.acinq.eclair.blockchain.fee
+
 import fr.acinq.bitcoin.Btc
-import fr.acinq.eclair.blockchain.rpc.BitcoinJsonRPCClient
+import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinJsonRPCClient
 import org.json4s.JsonAST.{JDouble, JInt}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -8,7 +9,7 @@ import scala.concurrent.{ExecutionContext, Future}
 /**
   * Created by PM on 09/07/2017.
   */
-class BitcoinCoreFeeProvider(rpcClient: BitcoinJsonRPCClient, defaultFeeratePerKB: Long)(implicit ec: ExecutionContext) extends FeeProvider {
+class BitcoinCoreFeeProvider(rpcClient: BitcoinJsonRPCClient, defaultFeerates: FeeratesPerByte)(implicit ec: ExecutionContext) extends FeeProvider {
 
   /**
     * We need this to keep commitment tx fees in sync with the state of the network
@@ -25,9 +26,19 @@ class BitcoinCoreFeeProvider(rpcClient: BitcoinJsonRPCClient, defaultFeeratePerK
       }
     })
 
-  override def getFeeratePerKB: Future[Long] = estimateSmartFee(3).map {
-    case f if f < 0 => defaultFeeratePerKB
-    case f => f
-  }
+  override def getFeerates: Future[FeeratesPerByte] = for {
+    block_1 <- estimateSmartFee(1)
+    blocks_2 <- estimateSmartFee(2)
+    blocks_6 <- estimateSmartFee(6)
+    blocks_12 <- estimateSmartFee(12)
+    blocks_36 <- estimateSmartFee(36)
+    blocks_72 <- estimateSmartFee(72)
+  } yield FeeratesPerByte(
+    block_1 = if (block_1 > 0) block_1 else defaultFeerates.block_1,
+    blocks_2 = if (blocks_2 > 0) blocks_2 else defaultFeerates.blocks_2,
+    blocks_6 = if (blocks_6 > 0) blocks_6 else defaultFeerates.blocks_6,
+    blocks_12 = if (blocks_12 > 0) blocks_12 else defaultFeerates.blocks_12,
+    blocks_36 = if (blocks_36 > 0) blocks_36 else defaultFeerates.blocks_36,
+    blocks_72 = if (blocks_72 > 0) blocks_72 else defaultFeerates.blocks_72)
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/BitpayInsightFeeProvider.scala

@@ -1,33 +0,0 @@
-package fr.acinq.eclair.blockchain.fee
-
-import akka.actor.ActorSystem
-import akka.http.scaladsl.Http
-import akka.http.scaladsl.model._
-import akka.http.scaladsl.unmarshalling.Unmarshal
-import akka.stream.ActorMaterializer
-import fr.acinq.bitcoin.{Btc, Satoshi}
-import org.json4s.JsonAST.{JDouble, JValue}
-import org.json4s.{DefaultFormats, jackson}
-import akka.http.scaladsl.unmarshalling.Unmarshal
-import akka.stream.ActorMaterializer
-import de.heikoseeberger.akkahttpjson4s.Json4sSupport._
-
-import scala.concurrent.{ExecutionContext, Future}
-
-/**
-  * Created by PM on 09/07/2017.
-  */
-class BitpayInsightFeeProvider(implicit system: ActorSystem, ec: ExecutionContext) extends FeeProvider {
-
-  implicit val materializer = ActorMaterializer()
-  val httpClient = Http(system)
-  implicit val serialization = jackson.Serialization
-  implicit val formats = DefaultFormats
-
-  override def getFeeratePerKB: Future[Long] =
-    for {
-      httpRes <- httpClient.singleRequest(HttpRequest(uri = Uri("https://test-insight.bitpay.com/api/utils/estimatefee?nbBlocks=3"), method = HttpMethods.GET))
-      json <- Unmarshal(httpRes).to[JValue]
-      JDouble(fee_per_kb) = json \ "3"
-    } yield (Btc(fee_per_kb): Satoshi).amount
-}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/ConstantFeeProvider.scala

@@ -1,10 +1,12 @@
 package fr.acinq.eclair.blockchain.fee
+
 import scala.concurrent.Future
 
 /**
   * Created by PM on 09/07/2017.
   */
-class ConstantFeeProvider(feeratePerKB: Long) extends FeeProvider {
+class ConstantFeeProvider(feerates: FeeratesPerByte) extends FeeProvider {
+
+  override def getFeerates: Future[FeeratesPerByte] = Future.successful(feerates)
 
-  override def getFeeratePerKB: Future[Long] = Future.successful(feeratePerKB)
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/EarnDotComFeeProvider.scala

@@ -0,0 +1,66 @@
+package fr.acinq.eclair.blockchain.fee
+
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import de.heikoseeberger.akkahttpjson4s.Json4sSupport._
+import org.json4s.JsonAST.{JArray, JInt, JValue}
+import org.json4s.{DefaultFormats, jackson}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+/**
+  * Created by PM on 16/11/2017.
+  */
+class EarnDotComFeeProvider(implicit system: ActorSystem, ec: ExecutionContext) extends FeeProvider {
+
+  import EarnDotComFeeProvider._
+
+  implicit val materializer = ActorMaterializer()
+  val httpClient = Http(system)
+  implicit val serialization = jackson.Serialization
+  implicit val formats = DefaultFormats
+
+  override def getFeerates: Future[FeeratesPerByte] =
+    for {
+      httpRes <- httpClient.singleRequest(HttpRequest(uri = Uri("https://bitcoinfees.earn.com/api/v1/fees/list"), method = HttpMethods.GET))
+      json <- Unmarshal(httpRes).to[JValue]
+      feeRanges = parseFeeRanges(json)
+    } yield extractFeerates(feeRanges)
+}
+
+object EarnDotComFeeProvider {
+
+  case class FeeRange(minFee: Long, maxFee: Long, memCount: Long, minDelay: Long, maxDelay: Long)
+
+  def parseFeeRanges(json: JValue): Seq[FeeRange] = {
+    val JArray(items) = json \ "fees"
+    items.map(item => {
+      val JInt(minFee) = item \ "minFee"
+      val JInt(maxFee) = item \ "maxFee"
+      val JInt(memCount) = item \ "memCount"
+      val JInt(minDelay) = item \ "minDelay"
+      val JInt(maxDelay) = item \ "maxDelay"
+      FeeRange(minFee = minFee.toLong, maxFee = maxFee.toLong, memCount = memCount.toLong, minDelay = minDelay.toLong, maxDelay = maxDelay.toLong)
+    })
+  }
+
+  def extractFeerate(feeRanges: Seq[FeeRange], maxBlockDelay: Int): Long = {
+    // first we keep only fee ranges with a max block delay below the limit
+    val belowLimit = feeRanges.filter(_.maxDelay <= maxBlockDelay)
+    // out of all the remaining fee ranges, we select the one with the minimum higher bound
+    belowLimit.minBy(_.maxFee).maxFee
+  }
+
+  def extractFeerates(feeRanges: Seq[FeeRange]): FeeratesPerByte =
+    FeeratesPerByte(
+      block_1 = extractFeerate(feeRanges, 1),
+      blocks_2 = extractFeerate(feeRanges, 2),
+      blocks_6 = extractFeerate(feeRanges, 6),
+      blocks_12 = extractFeerate(feeRanges, 12),
+      blocks_36 = extractFeerate(feeRanges, 36),
+      blocks_72 = extractFeerate(feeRanges, 72))
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/FallbackFeeProvider.scala

@@ -0,0 +1,20 @@
+package fr.acinq.eclair.blockchain.fee
+
+import scala.concurrent.{ExecutionContext, Future}
+
+/**
+  * This provider will try all child providers in sequence, until one of them works
+  */
+class FallbackFeeProvider(providers: Seq[FeeProvider])(implicit ec: ExecutionContext) extends FeeProvider {
+
+  require(providers.size >= 1, "need at least one fee provider")
+
+  def getFeerates(fallbacks: Seq[FeeProvider]): Future[FeeratesPerByte] =
+    fallbacks match {
+      case last +: Nil => last.getFeerates
+      case head +: remaining => head.getFeerates.recoverWith { case _ => getFeerates(remaining) }
+    }
+
+  override def getFeerates: Future[FeeratesPerByte] = getFeerates(providers)
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/FeeProvider.scala

@@ -1,5 +1,7 @@
 package fr.acinq.eclair.blockchain.fee
 
+import fr.acinq.eclair.feerateByte2Kw
+
 import scala.concurrent.Future
 
 /**
@@ -7,6 +9,34 @@ import scala.concurrent.Future
   */
 trait FeeProvider {
 
-  def getFeeratePerKB: Future[Long]
+  def getFeerates: Future[FeeratesPerByte]
 
 }
+
+case class FeeratesPerByte(block_1: Long, blocks_2: Long, blocks_6: Long, blocks_12: Long, blocks_36: Long, blocks_72: Long)
+
+case class FeeratesPerKw(block_1: Long, blocks_2: Long, blocks_6: Long, blocks_12: Long, blocks_36: Long, blocks_72: Long)
+
+object FeeratesPerKw {
+  def apply(feerates: FeeratesPerByte): FeeratesPerKw = FeeratesPerKw(
+    block_1 = feerateByte2Kw(feerates.block_1),
+    blocks_2 = feerateByte2Kw(feerates.blocks_2),
+    blocks_6 = feerateByte2Kw(feerates.blocks_6),
+    blocks_12 = feerateByte2Kw(feerates.blocks_12),
+    blocks_36 = feerateByte2Kw(feerates.blocks_36),
+    blocks_72 = feerateByte2Kw(feerates.blocks_72))
+
+  /**
+    * Used in tests
+    *
+    * @param feeratePerKw
+    * @return
+    */
+  def single(feeratePerKw: Long): FeeratesPerKw = FeeratesPerKw(
+    block_1 = feeratePerKw,
+    blocks_2 = feeratePerKw,
+    blocks_6 = feeratePerKw,
+    blocks_12 = feeratePerKw,
+    blocks_36 = feeratePerKw,
+    blocks_72 = feeratePerKw)
+}

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/wallet/EclairWallet.scala

@@ -1,22 +0,0 @@
-package fr.acinq.eclair.blockchain.wallet
-
-import fr.acinq.bitcoin.{BinaryData, Satoshi, Transaction}
-
-import scala.concurrent.Future
-
-/**
-  * Created by PM on 06/07/2017.
-  */
-trait EclairWallet {
-
-  def getBalance: Future[Satoshi]
-
-  def getFinalAddress: Future[String]
-
-  def makeFundingTx(pubkeyScript: BinaryData, amount: Satoshi, feeRatePerKw: Long): Future[MakeFundingTxResponse]
-
-  def commit(tx: Transaction): Future[Boolean]
-
-}
-
-final case class MakeFundingTxResponse(fundingTx: Transaction, fundingTxOutputIndex: Int)

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/zmq/ZeroMQEvents.scala

@@ -1,10 +0,0 @@
-package fr.acinq.eclair.blockchain.zmq
-
-/**
-  * Created by PM on 04/04/2017.
-  */
-sealed trait ZMQEvents
-
-case object ZMQConnected extends ZMQEvents
-
-case object ZMQDisconnected extends ZMQEvents

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelExceptions.scala

@@ -8,33 +8,36 @@ import fr.acinq.eclair.UInt64
   */
 
 class ChannelException(channelId: BinaryData, message: String) extends RuntimeException(message)
-
-case class DebugTriggeredException      (channelId: BinaryData) extends ChannelException(channelId, "debug-mode triggered failure")
-case class ClosingInProgress            (channelId: BinaryData) extends ChannelException(channelId, "cannot send new htlcs, closing in progress")
-case class ClosingAlreadyInProgress     (channelId: BinaryData) extends ChannelException(channelId, "closing already in progress")
-case class CannotCloseWithPendingChanges(channelId: BinaryData) extends ChannelException(channelId, "cannot close when there are pending changes")
-case class ChannelUnavailable           (channelId: BinaryData) extends ChannelException(channelId, "channel is unavailable (offline or closing)")
-case class InvalidFinalScript           (channelId: BinaryData) extends ChannelException(channelId, "invalid final script")
-case class HtlcTimedout                 (channelId: BinaryData) extends ChannelException(channelId, s"one or more htlcs timed out")
-case class FeerateTooDifferent          (channelId: BinaryData, localFeeratePerKw: Long, remoteFeeratePerKw: Long) extends ChannelException(channelId, s"local/remote feerates are too different: remoteFeeratePerKw=$remoteFeeratePerKw localFeeratePerKw=$localFeeratePerKw")
-case class InvalidCloseSignature        (channelId: BinaryData) extends ChannelException(channelId, "cannot verify their close signature")
-case class InvalidCommitmentSignature   (channelId: BinaryData) extends ChannelException(channelId, "invalid commitment signature")
-case class ForcedLocalCommit            (channelId: BinaryData, reason: String) extends ChannelException(channelId, s"forced local commit: reason")
-case class UnexpectedHtlcId             (channelId: BinaryData, expected: Long, actual: Long) extends ChannelException(channelId, s"unexpected htlc id: expected=$expected actual=$actual")
-case class InvalidPaymentHash           (channelId: BinaryData) extends ChannelException(channelId, "invalid payment hash")
-case class ExpiryTooSmall               (channelId: BinaryData, minimum: Long, actual: Long, blockCount: Long) extends ChannelException(channelId, s"expiry too small: required=$minimum actual=$actual blockCount=$blockCount")
-case class ExpiryCannotBeInThePast      (channelId: BinaryData, expiry: Long, blockCount: Long) extends ChannelException(channelId, s"expiry can't be in the past: expiry=$expiry blockCount=$blockCount")
-case class HtlcValueTooSmall            (channelId: BinaryData, minimum: Long, actual: Long) extends ChannelException(channelId, s"htlc value too small: mininmum=$minimum actual=$actual")
-case class HtlcValueTooHighInFlight     (channelId: BinaryData, maximum: UInt64, actual: UInt64) extends ChannelException(channelId, s"in-flight htlcs hold too much value: maximum=$maximum actual=$actual")
-case class TooManyAcceptedHtlcs         (channelId: BinaryData, maximum: Long) extends ChannelException(channelId, s"too many accepted htlcs: maximum=$maximum")
-case class InsufficientFunds            (channelId: BinaryData, amountMsat: Long, missingSatoshis: Long, reserveSatoshis: Long, feesSatoshis: Long) extends ChannelException(channelId, s"insufficient funds: missingSatoshis=$missingSatoshis reserveSatoshis=$reserveSatoshis fees=$feesSatoshis")
-case class InvalidHtlcPreimage          (channelId: BinaryData, id: Long) extends ChannelException(channelId, s"invalid htlc preimage for htlc id=$id")
-case class UnknownHtlcId                (channelId: BinaryData, id: Long) extends ChannelException(channelId, s"unknown htlc id=$id")
-case class FundeeCannotSendUpdateFee    (channelId: BinaryData) extends ChannelException(channelId, s"only the funder should send update_fee messages")
-case class CannotAffordFees             (channelId: BinaryData, missingSatoshis: Long, reserveSatoshis: Long, feesSatoshis: Long) extends ChannelException(channelId, s"can't pay the fee: missingSatoshis=$missingSatoshis reserveSatoshis=$reserveSatoshis feesSatoshis=$feesSatoshis")
-case class CannotSignWithoutChanges     (channelId: BinaryData) extends ChannelException(channelId, "cannot sign when there are no changes")
-case class CannotSignBeforeRevocation   (channelId: BinaryData) extends ChannelException(channelId, "cannot sign until next revocation hash is received")
-case class UnexpectedRevocation         (channelId: BinaryData) extends ChannelException(channelId, "received unexpected RevokeAndAck message")
-case class InvalidRevocation            (channelId: BinaryData) extends ChannelException(channelId, "invalid revocation")
-case class CommitmentSyncError          (channelId: BinaryData) extends ChannelException(channelId, "commitment sync error")
-case class RevocationSyncError          (channelId: BinaryData) extends ChannelException(channelId, "revocation sync error")
+// @formatter:off
+case class DebugTriggeredException             (channelId: BinaryData) extends ChannelException(channelId, "debug-mode triggered failure")
+case class ChannelReserveTooHigh               (channelId: BinaryData, channelReserveSatoshis: Long, reserveToFundingRatio: Double, maxReserveToFundingRatio: Double) extends ChannelException(channelId, s"channelReserveSatoshis too high: reserve=$channelReserveSatoshis fundingRatio=$reserveToFundingRatio maxFundingRatio=$maxReserveToFundingRatio")
+case class ClosingInProgress                   (channelId: BinaryData) extends ChannelException(channelId, "cannot send new htlcs, closing in progress")
+case class ClosingAlreadyInProgress            (channelId: BinaryData) extends ChannelException(channelId, "closing already in progress")
+case class CannotCloseWithUnsignedOutgoingHtlcs(channelId: BinaryData) extends ChannelException(channelId, "cannot close when there are unsigned outgoing htlcs")
+case class ChannelUnavailable                  (channelId: BinaryData) extends ChannelException(channelId, "channel is unavailable (offline or closing)")
+case class InvalidFinalScript                  (channelId: BinaryData) extends ChannelException(channelId, "invalid final script")
+case class HtlcTimedout                        (channelId: BinaryData) extends ChannelException(channelId, s"one or more htlcs timed out")
+case class FeerateTooDifferent                 (channelId: BinaryData, localFeeratePerKw: Long, remoteFeeratePerKw: Long) extends ChannelException(channelId, s"local/remote feerates are too different: remoteFeeratePerKw=$remoteFeeratePerKw localFeeratePerKw=$localFeeratePerKw")
+case class InvalidCloseSignature               (channelId: BinaryData) extends ChannelException(channelId, "cannot verify their close signature")
+case class InvalidCommitmentSignature          (channelId: BinaryData) extends ChannelException(channelId, "invalid commitment signature")
+case class ForcedLocalCommit                   (channelId: BinaryData, reason: String) extends ChannelException(channelId, s"forced local commit: reason")
+case class UnexpectedHtlcId                    (channelId: BinaryData, expected: Long, actual: Long) extends ChannelException(channelId, s"unexpected htlc id: expected=$expected actual=$actual")
+case class InvalidPaymentHash                  (channelId: BinaryData) extends ChannelException(channelId, "invalid payment hash")
+case class ExpiryTooSmall                      (channelId: BinaryData, minimum: Long, actual: Long, blockCount: Long) extends ChannelException(channelId, s"expiry too small: required=$minimum actual=$actual blockCount=$blockCount")
+case class ExpiryCannotBeInThePast             (channelId: BinaryData, expiry: Long, blockCount: Long) extends ChannelException(channelId, s"expiry can't be in the past: expiry=$expiry blockCount=$blockCount")
+case class HtlcValueTooSmall                   (channelId: BinaryData, minimum: Long, actual: Long) extends ChannelException(channelId, s"htlc value too small: minimum=$minimum actual=$actual")
+case class HtlcValueTooHighInFlight            (channelId: BinaryData, maximum: UInt64, actual: UInt64) extends ChannelException(channelId, s"in-flight htlcs hold too much value: maximum=$maximum actual=$actual")
+case class TooManyAcceptedHtlcs                (channelId: BinaryData, maximum: Long) extends ChannelException(channelId, s"too many accepted htlcs: maximum=$maximum")
+case class InsufficientFunds                   (channelId: BinaryData, amountMsat: Long, missingSatoshis: Long, reserveSatoshis: Long, feesSatoshis: Long) extends ChannelException(channelId, s"insufficient funds: missingSatoshis=$missingSatoshis reserveSatoshis=$reserveSatoshis fees=$feesSatoshis")
+case class InvalidHtlcPreimage                 (channelId: BinaryData, id: Long) extends ChannelException(channelId, s"invalid htlc preimage for htlc id=$id")
+case class UnknownHtlcId                       (channelId: BinaryData, id: Long) extends ChannelException(channelId, s"unknown htlc id=$id")
+case class FundeeCannotSendUpdateFee           (channelId: BinaryData) extends ChannelException(channelId, s"only the funder should send update_fee messages")
+case class CannotAffordFees                    (channelId: BinaryData, missingSatoshis: Long, reserveSatoshis: Long, feesSatoshis: Long) extends ChannelException(channelId, s"can't pay the fee: missingSatoshis=$missingSatoshis reserveSatoshis=$reserveSatoshis feesSatoshis=$feesSatoshis")
+case class CannotSignWithoutChanges            (channelId: BinaryData) extends ChannelException(channelId, "cannot sign when there are no changes")
+case class CannotSignBeforeRevocation          (channelId: BinaryData) extends ChannelException(channelId, "cannot sign until next revocation hash is received")
+case class UnexpectedRevocation                (channelId: BinaryData) extends ChannelException(channelId, "received unexpected RevokeAndAck message")
+case class InvalidRevocation                   (channelId: BinaryData) extends ChannelException(channelId, "invalid revocation")
+case class CommitmentSyncError                 (channelId: BinaryData) extends ChannelException(channelId, "commitment sync error")
+case class RevocationSyncError                 (channelId: BinaryData) extends ChannelException(channelId, "revocation sync error")
+case class InvalidFailureCode                  (channelId: BinaryData) extends ChannelException(channelId, "UpdateFailMalformedHtlc message doesn't have BADONION bit set")
+// @formatter:on

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelTypes.scala

@@ -2,7 +2,7 @@ package fr.acinq.eclair.channel
 
 import akka.actor.ActorRef
 import fr.acinq.bitcoin.Crypto.{Point, PrivateKey, PublicKey, Scalar}
-import fr.acinq.bitcoin.{BinaryData, Transaction}
+import fr.acinq.bitcoin.{BinaryData, OutPoint, Transaction}
 import fr.acinq.eclair.UInt64
 import fr.acinq.eclair.crypto.Sphinx
 import fr.acinq.eclair.transactions.CommitmentSpec
@@ -33,6 +33,7 @@ case object WAIT_FOR_ACCEPT_CHANNEL extends State
 case object WAIT_FOR_FUNDING_INTERNAL extends State
 case object WAIT_FOR_FUNDING_CREATED extends State
 case object WAIT_FOR_FUNDING_SIGNED extends State
+case object WAIT_FOR_FUNDING_PUBLISHED extends State
 case object WAIT_FOR_FUNDING_CONFIRMED extends State
 case object WAIT_FOR_FUNDING_LOCKED extends State
 case object NORMAL extends State
@@ -42,6 +43,7 @@ case object CLOSING extends State
 case object CLOSED extends State
 case object OFFLINE extends State
 case object SYNCING extends State
+case object ERR_FUNDING_PUBLISH_FAILED extends State
 case object ERR_FUNDING_LOST extends State
 case object ERR_FUNDING_TIMEOUT extends State
 case object ERR_INFORMATION_LEAK extends State
@@ -66,21 +68,16 @@ case class INPUT_RECONNECTED(remote: ActorRef)
 case class INPUT_RESTORED(data: HasCommitments)
 
 sealed trait BitcoinEvent
+case object BITCOIN_FUNDING_PUBLISH_FAILED extends BitcoinEvent
 case object BITCOIN_FUNDING_DEPTHOK extends BitcoinEvent
 case object BITCOIN_FUNDING_DEEPLYBURIED extends BitcoinEvent
 case object BITCOIN_FUNDING_LOST extends BitcoinEvent
 case object BITCOIN_FUNDING_TIMEOUT extends BitcoinEvent
 case object BITCOIN_FUNDING_SPENT extends BitcoinEvent
-case object BITCOIN_HTLC_SPENT extends BitcoinEvent
-case object BITCOIN_LOCALCOMMIT_DONE extends BitcoinEvent
-case object BITCOIN_REMOTECOMMIT_DONE extends BitcoinEvent
-case object BITCOIN_NEXTREMOTECOMMIT_DONE extends BitcoinEvent
-case object BITCOIN_PENALTY_DONE extends BitcoinEvent
-case object BITCOIN_CLOSE_DONE extends BitcoinEvent
-case class BITCOIN_FUNDING_OTHER_CHANNEL_SPENT(shortChannelId: Long) extends BitcoinEvent
+case object BITCOIN_OUTPUT_SPENT extends BitcoinEvent
 case class BITCOIN_TX_CONFIRMED(tx: Transaction) extends BitcoinEvent
+case class BITCOIN_FUNDING_EXTERNAL_CHANNEL_SPENT(shortChannelId: Long) extends BitcoinEvent
 case class BITCOIN_PARENT_TX_CONFIRMED(childTx: Transaction) extends BitcoinEvent
-case class BITCOIN_INPUT_SPENT(tx: Transaction) extends BitcoinEvent
 
 /*
        .d8888b.   .d88888b.  888b     d888 888b     d888        d8888 888b    888 8888888b.   .d8888b.
@@ -126,9 +123,9 @@ trait HasCommitments extends Data {
   def channelId = commitments.channelId
 }
 
-case class LocalCommitPublished(commitTx: Transaction, claimMainDelayedOutputTx: Option[Transaction], htlcSuccessTxs: List[Transaction], htlcTimeoutTxs: List[Transaction], claimHtlcDelayedTx: List[Transaction])
-case class RemoteCommitPublished(commitTx: Transaction, claimMainOutputTx: Option[Transaction], claimHtlcSuccessTxs: List[Transaction], claimHtlcTimeoutTxs: List[Transaction])
-case class RevokedCommitPublished(commitTx: Transaction, claimMainOutputTx: Option[Transaction], mainPenaltyTx: Option[Transaction], claimHtlcTimeoutTxs: List[Transaction], htlcTimeoutTxs: List[Transaction], htlcPenaltyTxs: List[Transaction])
+case class LocalCommitPublished(commitTx: Transaction, claimMainDelayedOutputTx: Option[Transaction], htlcSuccessTxs: List[Transaction], htlcTimeoutTxs: List[Transaction], claimHtlcDelayedTx: List[Transaction], spent: Map[OutPoint, BinaryData])
+case class RemoteCommitPublished(commitTx: Transaction, claimMainOutputTx: Option[Transaction], claimHtlcSuccessTxs: List[Transaction], claimHtlcTimeoutTxs: List[Transaction], spent: Map[OutPoint, BinaryData])
+case class RevokedCommitPublished(commitTx: Transaction, claimMainOutputTx: Option[Transaction], mainPenaltyTx: Option[Transaction], claimHtlcTimeoutTxs: List[Transaction], htlcTimeoutTxs: List[Transaction], htlcPenaltyTxs: List[Transaction], spent: Map[OutPoint, BinaryData])
 
 final case class DATA_WAIT_FOR_OPEN_CHANNEL(initFundee: INPUT_INIT_FUNDEE) extends Data
 final case class DATA_WAIT_FOR_ACCEPT_CHANNEL(initFunder: INPUT_INIT_FUNDER, lastSent: OpenChannel) extends Data
@@ -164,8 +161,9 @@ final case class LocalParams(nodeId: PublicKey,
                              maxAcceptedHtlcs: Int,
                              fundingPrivKey: PrivateKey,
                              revocationSecret: Scalar,
-                             paymentKey: PrivateKey,
+                             paymentKey: Scalar,
                              delayedPaymentKey: Scalar,
+                             htlcKey: Scalar,
                              defaultFinalScriptPubKey: BinaryData,
                              shaSeed: BinaryData,
                              isFunder: Boolean,
@@ -175,6 +173,7 @@ final case class LocalParams(nodeId: PublicKey,
   val paymentBasepoint = paymentKey.toPoint
   val delayedPaymentBasepoint = delayedPaymentKey.toPoint
   val revocationBasepoint = revocationSecret.toPoint
+  val htlcBasepoint = htlcKey.toPoint
 }
 
 final case class RemoteParams(nodeId: PublicKey,
@@ -188,6 +187,7 @@ final case class RemoteParams(nodeId: PublicKey,
                               revocationBasepoint: Point,
                               paymentBasepoint: Point,
                               delayedPaymentBasepoint: Point,
+                              htlcBasepoint: Point,
                               globalFeatures: BinaryData,
                               localFeatures: BinaryData)
 

eclair-core/src/main/scala/fr/acinq/eclair/channel/Commitments.scala

@@ -2,11 +2,12 @@ package fr.acinq.eclair.channel
 
 import fr.acinq.bitcoin.Crypto.{Point, PrivateKey, sha256}
 import fr.acinq.bitcoin.{BinaryData, Crypto, Satoshi, Transaction}
-import fr.acinq.eclair.{Globals, UInt64}
 import fr.acinq.eclair.crypto.{Generators, ShaChain, Sphinx}
+import fr.acinq.eclair.payment.Origin
 import fr.acinq.eclair.transactions.Transactions._
 import fr.acinq.eclair.transactions._
 import fr.acinq.eclair.wire._
+import fr.acinq.eclair.{Globals, UInt64}
 import grizzled.slf4j.Logging
 
 // @formatter:off
@@ -35,11 +36,12 @@ case class Commitments(localParams: LocalParams, remoteParams: RemoteParams,
                        localCommit: LocalCommit, remoteCommit: RemoteCommit,
                        localChanges: LocalChanges, remoteChanges: RemoteChanges,
                        localNextHtlcId: Long, remoteNextHtlcId: Long,
+                       originChannels: Map[Long, Origin], // for outgoing htlcs relayed through us, the id of the previous channel
                        remoteNextCommitInfo: Either[WaitingForRevocation, Point],
                        commitInput: InputInfo,
                        remotePerCommitmentSecrets: ShaChain, channelId: BinaryData) {
 
-  def hasNoPendingHtlcs: Boolean = localCommit.spec.htlcs.isEmpty && remoteCommit.spec.htlcs.isEmpty
+  def hasNoPendingHtlcs: Boolean = localCommit.spec.htlcs.isEmpty && remoteCommit.spec.htlcs.isEmpty && remoteNextCommitInfo.isRight
 
   def hasTimedoutOutgoingHtlcs(blockheight: Long): Boolean =
     localCommit.spec.htlcs.exists(htlc => htlc.direction == OUT && blockheight >= htlc.add.expiry) ||
@@ -72,7 +74,7 @@ object Commitments extends Logging {
     * @param cmd         add HTLC command
     * @return either Left(failure, error message) where failure is a failure message (see BOLT #4 and the Failure Message class) or Right((new commitments, updateAddHtlc)
     */
-  def sendAdd(commitments: Commitments, cmd: CMD_ADD_HTLC): Either[ChannelException, (Commitments, UpdateAddHtlc)] = {
+  def sendAdd(commitments: Commitments, cmd: CMD_ADD_HTLC, origin: Origin): Either[ChannelException, (Commitments, UpdateAddHtlc)] = {
 
     if (cmd.paymentHash.size != 32) {
       return Left(InvalidPaymentHash(commitments.channelId))
@@ -89,8 +91,11 @@ object Commitments extends Logging {
 
     // let's compute the current commitment *as seen by them* with this change taken into account
     val add = UpdateAddHtlc(commitments.channelId, commitments.localNextHtlcId, cmd.amountMsat, cmd.paymentHash, cmd.expiry, cmd.onion)
-    val commitments1 = addLocalProposal(commitments, add).copy(localNextHtlcId = commitments.localNextHtlcId + 1)
-    val reduced = CommitmentSpec.reduce(commitments1.remoteCommit.spec, commitments1.remoteChanges.acked, commitments1.localChanges.proposed)
+    // we increment the local htlc index and add an entry to the origins map
+    val commitments1 = addLocalProposal(commitments, add).copy(localNextHtlcId = commitments.localNextHtlcId + 1, originChannels = commitments.originChannels + (add.id -> origin))
+    // we need to base the next current commitment on the last sig we sent, even if we didn't yet receive their revocation
+    val remoteCommit1 = commitments1.remoteNextCommitInfo.left.toOption.map(_.nextRemoteCommit).getOrElse(commitments1.remoteCommit)
+    val reduced = CommitmentSpec.reduce(remoteCommit1.spec, commitments1.remoteChanges.acked, commitments1.localChanges.proposed)
 
     val htlcValueInFlight = UInt64(reduced.htlcs.map(_.add.amountMsat).sum)
     if (htlcValueInFlight > commitments1.remoteParams.maxHtlcValueInFlightMsat) {
@@ -187,9 +192,9 @@ object Commitments extends Logging {
       case None => throw UnknownHtlcId(commitments.channelId, cmd.id)
     }
 
-  def receiveFulfill(commitments: Commitments, fulfill: UpdateFulfillHtlc): Either[Commitments, Commitments] =
+  def receiveFulfill(commitments: Commitments, fulfill: UpdateFulfillHtlc): Either[Commitments, (Commitments, Origin)] =
     getHtlcCrossSigned(commitments, OUT, fulfill.id) match {
-      case Some(htlc) if htlc.paymentHash == sha256(fulfill.paymentPreimage) => Right(addRemoteProposal(commitments, fulfill))
+      case Some(htlc) if htlc.paymentHash == sha256(fulfill.paymentPreimage) => Right((addRemoteProposal(commitments, fulfill), commitments.originChannels(fulfill.id)))
       case Some(htlc) => throw InvalidHtlcPreimage(commitments.channelId, fulfill.id)
       case None => throw UnknownHtlcId(commitments.channelId, fulfill.id)
     }
@@ -217,7 +222,11 @@ object Commitments extends Logging {
       case None => throw UnknownHtlcId(commitments.channelId, cmd.id)
     }
 
-  def sendFailMalformed(commitments: Commitments, cmd: CMD_FAIL_MALFORMED_HTLC): (Commitments, UpdateFailMalformedHtlc) =
+  def sendFailMalformed(commitments: Commitments, cmd: CMD_FAIL_MALFORMED_HTLC): (Commitments, UpdateFailMalformedHtlc) = {
+    // BADONION bit must be set in failure_code
+    if ((cmd.failureCode & FailureMessageCodecs.BADONION) == 0) {
+      throw InvalidFailureCode(commitments.channelId)
+    }
     getHtlcCrossSigned(commitments, IN, cmd.id) match {
       case Some(htlc) if commitments.localChanges.proposed.exists {
         case u: UpdateFulfillHtlc if htlc.id == u.id => true
@@ -233,18 +242,25 @@ object Commitments extends Logging {
         (commitments1, fail)
       case None => throw UnknownHtlcId(commitments.channelId, cmd.id)
     }
+  }
 
-  def receiveFail(commitments: Commitments, fail: UpdateFailHtlc): Either[Commitments, Commitments] =
+  def receiveFail(commitments: Commitments, fail: UpdateFailHtlc): Either[Commitments, (Commitments, Origin)] =
     getHtlcCrossSigned(commitments, OUT, fail.id) match {
-      case Some(htlc) => Right(addRemoteProposal(commitments, fail))
+      case Some(htlc) => Right((addRemoteProposal(commitments, fail), commitments.originChannels(fail.id)))
       case None => throw UnknownHtlcId(commitments.channelId, fail.id)
     }
 
-  def receiveFailMalformed(commitments: Commitments, fail: UpdateFailMalformedHtlc): Either[Commitments, Commitments] =
+  def receiveFailMalformed(commitments: Commitments, fail: UpdateFailMalformedHtlc): Either[Commitments, (Commitments, Origin)] = {
+    // A receiving node MUST fail the channel if the BADONION bit in failure_code is not set for update_fail_malformed_htlc.
+    if ((fail.failureCode & FailureMessageCodecs.BADONION) == 0) {
+      throw InvalidFailureCode(commitments.channelId)
+    }
+
     getHtlcCrossSigned(commitments, OUT, fail.id) match {
-      case Some(htlc) => Right(addRemoteProposal(commitments, fail))
+      case Some(htlc) => Right((addRemoteProposal(commitments, fail), commitments.originChannels(fail.id)))
       case None => throw UnknownHtlcId(commitments.channelId, fail.id)
     }
+  }
 
   def sendFee(commitments: Commitments, cmd: CMD_UPDATE_FEE): (Commitments, UpdateFee) = {
     if (!commitments.localParams.isFunder) {
@@ -271,7 +287,7 @@ object Commitments extends Logging {
       throw FundeeCannotSendUpdateFee(commitments.channelId)
     }
 
-    val localFeeratePerKw = Globals.feeratePerKw.get()
+    val localFeeratePerKw = Globals.feeratesPerKw.get.block_1
     if (Helpers.isFeeDiffTooHigh(fee.feeratePerKw, localFeeratePerKw, maxFeerateMismatch)) {
       throw FeerateTooDifferent(commitments.channelId, localFeeratePerKw = localFeeratePerKw, remoteFeeratePerKw = fee.feeratePerKw)
     }
@@ -295,6 +311,10 @@ object Commitments extends Logging {
     commitments1
   }
 
+  def localHasUnsignedOutgoingHtlcs(commitments: Commitments): Boolean = commitments.localChanges.proposed.collectFirst { case u: UpdateAddHtlc => u }.isDefined
+
+  def remoteHasUnsignedOutgoingHtlcs(commitments: Commitments): Boolean = commitments.remoteChanges.proposed.collectFirst { case u: UpdateAddHtlc => u }.isDefined
+
   def localHasChanges(commitments: Commitments): Boolean = commitments.remoteChanges.acked.size > 0 || commitments.localChanges.proposed.size > 0
 
   def remoteHasChanges(commitments: Commitments): Boolean = commitments.localChanges.acked.size > 0 || commitments.remoteChanges.proposed.size > 0
@@ -315,8 +335,8 @@ object Commitments extends Logging {
         val sig = Transactions.sign(remoteCommitTx, localParams.fundingPrivKey)
 
         val sortedHtlcTxs: Seq[TransactionWithInputInfo] = (htlcTimeoutTxs ++ htlcSuccessTxs).sortBy(_.input.outPoint.index)
-        val paymentKey = Generators.derivePrivKey(localParams.paymentKey, remoteNextPerCommitmentPoint)
-        val htlcSigs = sortedHtlcTxs.map(Transactions.sign(_, paymentKey))
+        val htlcKey = Generators.derivePrivKey(localParams.htlcKey, remoteNextPerCommitmentPoint)
+        val htlcSigs = sortedHtlcTxs.map(Transactions.sign(_, htlcKey))
 
         // don't sign if they don't get paid
         val commitSig = CommitSig(
@@ -340,7 +360,7 @@ object Commitments extends Logging {
     // they sent us a signature for *their* view of *our* next commit tx
     // so in terms of rev.hashes and indexes we have:
     // ourCommit.index -> our current revocation hash, which is about to become our old revocation hash
-    // ourCommit.index + 1 -> our next revocation hash, used by * them * to build the sig we've just received, and which
+    // ourCommit.index + 1 -> our next revocation hash, used by *them* to build the sig we've just received, and which
     // is about to become our current revocation hash
     // ourCommit.index + 2 -> which is about to become our next revocation hash
     // we will reply to this sig with our old revocation hash preimage (at index) and our next revocation hash (at index + 1)
@@ -368,9 +388,9 @@ object Commitments extends Logging {
 
     val sortedHtlcTxs: Seq[TransactionWithInputInfo] = (htlcTimeoutTxs ++ htlcSuccessTxs).sortBy(_.input.outPoint.index)
     require(commit.htlcSignatures.size == sortedHtlcTxs.size, s"htlc sig count mismatch (received=${commit.htlcSignatures.size}, expected=${sortedHtlcTxs.size})")
-    val localPaymentKey = Generators.derivePrivKey(localParams.paymentKey, localPerCommitmentPoint)
-    val htlcSigs = sortedHtlcTxs.map(Transactions.sign(_, localPaymentKey))
-    val remotePaymentPubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, localPerCommitmentPoint)
+    val localHtlcKey = Generators.derivePrivKey(localParams.htlcKey, localPerCommitmentPoint)
+    val htlcSigs = sortedHtlcTxs.map(Transactions.sign(_, localHtlcKey))
+    val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, localPerCommitmentPoint)
     // combine the sigs to make signed txes
     val htlcTxsAndSigs = (sortedHtlcTxs, htlcSigs, commit.htlcSignatures).zipped.toList.collect {
       case (htlcTx: HtlcTimeoutTx, localSig, remoteSig) =>
@@ -378,7 +398,7 @@ object Commitments extends Logging {
         HtlcTxAndSigs(htlcTx, localSig, remoteSig)
       case (htlcTx: HtlcSuccessTx, localSig, remoteSig) =>
         // we can't check that htlc-success tx are spendable because we need the payment preimage; thus we only check the remote sig
-        require(Transactions.checkSig(htlcTx, remoteSig, remotePaymentPubkey), "bad sig")
+        require(Transactions.checkSig(htlcTx, remoteSig, remoteHtlcPubkey), "bad sig")
         HtlcTxAndSigs(htlcTx, localSig, remoteSig)
     }
 
@@ -392,15 +412,19 @@ object Commitments extends Logging {
     )
 
     // update our commitment data
-    val ourCommit1 = LocalCommit(
+    val localCommit1 = LocalCommit(
       index = localCommit.index + 1,
       spec,
       publishableTxs = PublishableTxs(signedCommitTx, htlcTxsAndSigs))
     val ourChanges1 = localChanges.copy(acked = Nil)
     val theirChanges1 = remoteChanges.copy(proposed = Nil, acked = remoteChanges.acked ++ remoteChanges.proposed)
-    val commitments1 = commitments.copy(localCommit = ourCommit1, localChanges = ourChanges1, remoteChanges = theirChanges1)
+    // the outgoing following htlcs have been completed (fulfilled or failed) when we received this sig
+    val completedOutgoingHtlcs = commitments.localCommit.spec.htlcs.filter(_.direction == OUT).map(_.add.id) -- localCommit1.spec.htlcs.filter(_.direction == OUT).map(_.add.id)
+    // we remove the newly completed htlcs from the origin map
+    val originChannels1 = commitments.originChannels -- completedOutgoingHtlcs
+    val commitments1 = commitments.copy(localCommit = localCommit1, localChanges = ourChanges1, remoteChanges = theirChanges1, originChannels = originChannels1)
 
-    logger.debug(s"current commit: index=${ourCommit1.index} htlc_in=${ourCommit1.spec.htlcs.filter(_.direction == IN).size} htlc_out=${ourCommit1.spec.htlcs.filter(_.direction == OUT).size} txid=${ourCommit1.publishableTxs.commitTx.tx.txid} tx=${Transaction.write(ourCommit1.publishableTxs.commitTx.tx)}")
+    logger.debug(s"current commit: index=${localCommit1.index} htlc_in=${localCommit1.spec.htlcs.filter(_.direction == IN).size} htlc_out=${localCommit1.spec.htlcs.filter(_.direction == OUT).size} txid=${localCommit1.publishableTxs.commitTx.tx.txid} tx=${Transaction.write(localCommit1.publishableTxs.commitTx.tx)}")
 
     (commitments1, revocation)
   }
@@ -426,22 +450,26 @@ object Commitments extends Logging {
   }
 
   def makeLocalTxs(commitTxNumber: Long, localParams: LocalParams, remoteParams: RemoteParams, commitmentInput: InputInfo, localPerCommitmentPoint: Point, spec: CommitmentSpec): (CommitTx, Seq[HtlcTimeoutTx], Seq[HtlcSuccessTx]) = {
-    val localPubkey = Generators.derivePubKey(localParams.paymentBasepoint, localPerCommitmentPoint)
-    val localDelayedPubkey = Generators.derivePubKey(localParams.delayedPaymentBasepoint, localPerCommitmentPoint)
-    val remotePubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, localPerCommitmentPoint)
+    val localPaymentPubkey = Generators.derivePubKey(localParams.paymentBasepoint, localPerCommitmentPoint)
+    val localDelayedPaymentPubkey = Generators.derivePubKey(localParams.delayedPaymentBasepoint, localPerCommitmentPoint)
+    val localHtlcPubkey = Generators.derivePubKey(localParams.htlcBasepoint, localPerCommitmentPoint)
+    val remotePaymentPubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, localPerCommitmentPoint)
+    val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, localPerCommitmentPoint)
     val localRevocationPubkey = Generators.revocationPubKey(remoteParams.revocationBasepoint, localPerCommitmentPoint)
-    val commitTx = Transactions.makeCommitTx(commitmentInput, commitTxNumber, localParams.paymentBasepoint, remoteParams.paymentBasepoint, localParams.isFunder, Satoshi(localParams.dustLimitSatoshis), localPubkey, localRevocationPubkey, remoteParams.toSelfDelay, localDelayedPubkey, remotePubkey, spec)
-    val (htlcTimeoutTxs, htlcSuccessTxs) = Transactions.makeHtlcTxs(commitTx.tx, Satoshi(localParams.dustLimitSatoshis), localRevocationPubkey, remoteParams.toSelfDelay, localPubkey, localDelayedPubkey, remotePubkey, spec)
+    val commitTx = Transactions.makeCommitTx(commitmentInput, commitTxNumber, localParams.paymentBasepoint, remoteParams.paymentBasepoint, localParams.isFunder, Satoshi(localParams.dustLimitSatoshis), localRevocationPubkey, remoteParams.toSelfDelay, localDelayedPaymentPubkey, remotePaymentPubkey, localHtlcPubkey, remoteHtlcPubkey, spec)
+    val (htlcTimeoutTxs, htlcSuccessTxs) = Transactions.makeHtlcTxs(commitTx.tx, Satoshi(localParams.dustLimitSatoshis), localRevocationPubkey, remoteParams.toSelfDelay, localDelayedPaymentPubkey, localHtlcPubkey, remoteHtlcPubkey, spec)
     (commitTx, htlcTimeoutTxs, htlcSuccessTxs)
   }
 
   def makeRemoteTxs(commitTxNumber: Long, localParams: LocalParams, remoteParams: RemoteParams, commitmentInput: InputInfo, remotePerCommitmentPoint: Point, spec: CommitmentSpec): (CommitTx, Seq[HtlcTimeoutTx], Seq[HtlcSuccessTx]) = {
-    val localPubkey = Generators.derivePubKey(localParams.paymentBasepoint, remotePerCommitmentPoint)
-    val remotePubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, remotePerCommitmentPoint)
-    val remoteDelayedPubkey = Generators.derivePubKey(remoteParams.delayedPaymentBasepoint, remotePerCommitmentPoint)
+    val localPaymentPubkey = Generators.derivePubKey(localParams.paymentBasepoint, remotePerCommitmentPoint)
+    val localHtlcPubkey = Generators.derivePubKey(localParams.htlcBasepoint, remotePerCommitmentPoint)
+    val remotePaymentPubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, remotePerCommitmentPoint)
+    val remoteDelayedPaymentPubkey = Generators.derivePubKey(remoteParams.delayedPaymentBasepoint, remotePerCommitmentPoint)
+    val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, remotePerCommitmentPoint)
     val remoteRevocationPubkey = Generators.revocationPubKey(localParams.revocationBasepoint, remotePerCommitmentPoint)
-    val commitTx = Transactions.makeCommitTx(commitmentInput, commitTxNumber, remoteParams.paymentBasepoint, localParams.paymentBasepoint, !localParams.isFunder, Satoshi(remoteParams.dustLimitSatoshis), remotePubkey, remoteRevocationPubkey, localParams.toSelfDelay, remoteDelayedPubkey, localPubkey, spec)
-    val (htlcTimeoutTxs, htlcSuccessTxs) = Transactions.makeHtlcTxs(commitTx.tx, Satoshi(remoteParams.dustLimitSatoshis), remoteRevocationPubkey, localParams.toSelfDelay, remotePubkey, remoteDelayedPubkey, localPubkey, spec)
+    val commitTx = Transactions.makeCommitTx(commitmentInput, commitTxNumber, remoteParams.paymentBasepoint, localParams.paymentBasepoint, !localParams.isFunder, Satoshi(remoteParams.dustLimitSatoshis), remoteRevocationPubkey, localParams.toSelfDelay, remoteDelayedPaymentPubkey, localPaymentPubkey, remoteHtlcPubkey, localHtlcPubkey, spec)
+    val (htlcTimeoutTxs, htlcSuccessTxs) = Transactions.makeHtlcTxs(commitTx.tx, Satoshi(remoteParams.dustLimitSatoshis), remoteRevocationPubkey, localParams.toSelfDelay, remoteDelayedPaymentPubkey, remoteHtlcPubkey, localHtlcPubkey, spec)
     (commitTx, htlcTimeoutTxs, htlcSuccessTxs)
   }
 

eclair-core/src/main/scala/fr/acinq/eclair/channel/Forwarder.scala

@@ -2,7 +2,7 @@ package fr.acinq.eclair.channel
 
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import fr.acinq.eclair.NodeParams
-import fr.acinq.eclair.wire.{Error, LightningMessage}
+import fr.acinq.eclair.wire.LightningMessage
 
 /**
   * Created by fabrice on 27/02/17.

eclair-core/src/main/scala/fr/acinq/eclair/channel/Helpers.scala

@@ -3,12 +3,13 @@ package fr.acinq.eclair.channel
 import fr.acinq.bitcoin.Crypto.{Point, PublicKey, Scalar, sha256}
 import fr.acinq.bitcoin.Script._
 import fr.acinq.bitcoin.{OutPoint, _}
-import fr.acinq.eclair.blockchain.wallet.EclairWallet
+import fr.acinq.eclair.blockchain.EclairWallet
 import fr.acinq.eclair.crypto.Generators
+import fr.acinq.eclair.router.Announcements
 import fr.acinq.eclair.transactions.Scripts._
 import fr.acinq.eclair.transactions.Transactions._
 import fr.acinq.eclair.transactions._
-import fr.acinq.eclair.wire.{ClosingSigned, UpdateAddHtlc, UpdateFulfillHtlc}
+import fr.acinq.eclair.wire.{AnnouncementSignatures, ClosingSigned, UpdateAddHtlc, UpdateFulfillHtlc}
 import fr.acinq.eclair.{Globals, NodeParams}
 import grizzled.slf4j.Logging
 
@@ -37,14 +38,21 @@ object Helpers {
     case d: HasCommitments => d.channelId
   }
 
-  def validateParamsFunder(nodeParams: NodeParams, channelReserveSatoshis: Long, fundingSatoshis: Long): Unit = {
+  def validateParamsFunder(temporaryChannelId: BinaryData, nodeParams: NodeParams, channelReserveSatoshis: Long, fundingSatoshis: Long): Unit = {
     val reserveToFundingRatio = channelReserveSatoshis.toDouble / fundingSatoshis
-    require(reserveToFundingRatio <= nodeParams.maxReserveToFundingRatio, s"channelReserveSatoshis too high: ratio=$reserveToFundingRatio max=${nodeParams.maxReserveToFundingRatio}")
+    if (reserveToFundingRatio > nodeParams.maxReserveToFundingRatio) {
+      throw new ChannelReserveTooHigh(temporaryChannelId, channelReserveSatoshis, reserveToFundingRatio, nodeParams.maxReserveToFundingRatio)
+    }
   }
 
-  def validateParamsFundee(nodeParams: NodeParams, channelReserveSatoshis: Long, fundingSatoshis: Long, chainHash: BinaryData): Unit = {
+  def validateParamsFundee(temporaryChannelId: BinaryData, nodeParams: NodeParams, channelReserveSatoshis: Long, fundingSatoshis: Long, chainHash: BinaryData, initialFeeratePerKw: Long): Unit = {
     require(nodeParams.chainHash == chainHash, s"invalid chain hash $chainHash (we are on ${nodeParams.chainHash})")
-    validateParamsFunder(nodeParams, channelReserveSatoshis, fundingSatoshis)
+    val localFeeratePerKw = Globals.feeratesPerKw.get.block_1
+    // we are fundee => initialFeeratePerKw has been set by remote
+    if (isFeeDiffTooHigh(initialFeeratePerKw, localFeeratePerKw, nodeParams.maxFeerateMismatch)) {
+      throw new FeerateTooDifferent(temporaryChannelId, localFeeratePerKw, initialFeeratePerKw)
+    }
+    validateParamsFunder(temporaryChannelId, nodeParams, channelReserveSatoshis, fundingSatoshis)
   }
 
   /**
@@ -73,6 +81,13 @@ object Helpers {
     remoteFeeratePerKw > 0 && feeRateMismatch(remoteFeeratePerKw, localFeeratePerKw) > maxFeerateMismatchRatio
   }
 
+  def makeAnnouncementSignatures(nodeParams: NodeParams, commitments: Commitments, shortChannelId: Long) = {
+    // TODO: empty features
+    val features = BinaryData("")
+    val (localNodeSig, localBitcoinSig) = Announcements.signChannelAnnouncement(nodeParams.chainHash, shortChannelId, nodeParams.privateKey, commitments.remoteParams.nodeId, commitments.localParams.fundingPrivKey, commitments.remoteParams.fundingPubKey, features)
+    AnnouncementSignatures(commitments.channelId, shortChannelId, localNodeSig, localBitcoinSig)
+  }
+
   def getFinalScriptPubKey(wallet: EclairWallet): BinaryData = {
     import scala.concurrent.duration._
     val finalAddress = Await.result(wallet.getFinalAddress, 40 seconds)
@@ -102,24 +117,21 @@ object Helpers {
       * @param remoteFirstPerCommitmentPoint
       * @return (localSpec, localTx, remoteSpec, remoteTx, fundingTxOutput)
       */
-    def makeFirstCommitTxs(localParams: LocalParams, remoteParams: RemoteParams, fundingSatoshis: Long, pushMsat: Long, initialFeeratePerKw: Long, fundingTxHash: BinaryData, fundingTxOutputIndex: Int, remoteFirstPerCommitmentPoint: Point, maxFeerateMismatch: Double): (CommitmentSpec, CommitTx, CommitmentSpec, CommitTx) = {
+    def makeFirstCommitTxs(temporaryChannelId: BinaryData, localParams: LocalParams, remoteParams: RemoteParams, fundingSatoshis: Long, pushMsat: Long, initialFeeratePerKw: Long, fundingTxHash: BinaryData, fundingTxOutputIndex: Int, remoteFirstPerCommitmentPoint: Point, maxFeerateMismatch: Double): (CommitmentSpec, CommitTx, CommitmentSpec, CommitTx) = {
       val toLocalMsat = if (localParams.isFunder) fundingSatoshis * 1000 - pushMsat else pushMsat
       val toRemoteMsat = if (localParams.isFunder) pushMsat else fundingSatoshis * 1000 - pushMsat
 
-      val localSpec = CommitmentSpec(Set.empty[Htlc], feeratePerKw = initialFeeratePerKw, toLocalMsat = toLocalMsat, toRemoteMsat = toRemoteMsat)
-      val remoteSpec = CommitmentSpec(Set.empty[Htlc], feeratePerKw = initialFeeratePerKw, toLocalMsat = toRemoteMsat, toRemoteMsat = toLocalMsat)
+      val localSpec = CommitmentSpec(Set.empty[DirectedHtlc], feeratePerKw = initialFeeratePerKw, toLocalMsat = toLocalMsat, toRemoteMsat = toRemoteMsat)
+      val remoteSpec = CommitmentSpec(Set.empty[DirectedHtlc], feeratePerKw = initialFeeratePerKw, toLocalMsat = toRemoteMsat, toRemoteMsat = toLocalMsat)
 
-      // TODO: should we check the fees sooner in the process?
       if (!localParams.isFunder) {
-        // they are funder, we need to make sure that they can pay the fee is reasonable, and that they can afford to pay it
-        val localFeeratePerKw = Globals.feeratePerKw.get()
-        if (isFeeDiffTooHigh(initialFeeratePerKw, localFeeratePerKw, maxFeerateMismatch)) {
-          throw new RuntimeException(s"local/remote feerates are too different: remoteFeeratePerKw=$initialFeeratePerKw localFeeratePerKw=$localFeeratePerKw")
+        // they are funder, therefore they pay the fee: we need to make sure they can afford it!
+        val toRemoteMsat = remoteSpec.toLocalMsat
+        val fees = Transactions.commitTxFee(Satoshi(remoteParams.dustLimitSatoshis), remoteSpec).amount
+        val missing = toRemoteMsat / 1000 - localParams.channelReserveSatoshis - fees
+        if (missing < 0) {
+          throw CannotAffordFees(temporaryChannelId, missingSatoshis = -1 * missing, reserveSatoshis = localParams.channelReserveSatoshis, feesSatoshis = fees)
         }
-        val toRemote = MilliSatoshi(remoteSpec.toLocalMsat)
-        val reserve = Satoshi(localParams.channelReserveSatoshis)
-        val fees = Transactions.commitTxFee(Satoshi(remoteParams.dustLimitSatoshis), remoteSpec)
-        require(toRemote >= reserve + fees, s"remote cannot pay the fees for the initial commit tx: toRemote=$toRemote reserve=$reserve fees=$fees")
       }
 
       val commitmentInput = makeFundingInputInfo(fundingTxHash, fundingTxOutputIndex, Satoshi(fundingSatoshis), localParams.fundingPrivKey.publicKey, remoteParams.fundingPubKey)
@@ -151,8 +163,8 @@ object Helpers {
         // this is just to estimate the weight, it depends on size of the pubkey scripts
         val dummyClosingTx = Transactions.makeClosingTx(commitInput, localScriptPubkey, remoteScriptPubkey, localParams.isFunder, Satoshi(0), Satoshi(0), localCommit.spec)
         val closingWeight = Transaction.weight(Transactions.addSigs(dummyClosingTx, localParams.fundingPrivKey.publicKey, remoteParams.fundingPubKey, "aa" * 71, "bb" * 71).tx)
-        // we use our local fee estimate
-        val feeratePerKw = Globals.feeratePerKw.get()
+        // no need to use a very high fee here
+        val feeratePerKw = Globals.feeratesPerKw.get.blocks_6
         logger.info(s"using feeratePerKw=$feeratePerKw for closing tx")
         Transactions.weight2fee(feeratePerKw, closingWeight)
       }
@@ -209,12 +221,12 @@ object Helpers {
       val localRevocationPubkey = Generators.revocationPubKey(remoteParams.revocationBasepoint, localPerCommitmentPoint)
       val localDelayedPrivkey = Generators.derivePrivKey(localParams.delayedPaymentKey, localPerCommitmentPoint)
 
-      // for now we use the current commit's fee rate, it should be up-to-date
-      val feeratePerKw = localCommit.spec.feeratePerKw
+      // no need to use a high fee rate for delayed transactions (we are the only one who can spend them)
+      val feeratePerKwDelayed = Globals.feeratesPerKw.get.blocks_6
 
       // first we will claim our main output as soon as the delay is over
       val mainDelayedTx = generateTx("main-delayed-output")(Try {
-        val claimDelayed = Transactions.makeClaimDelayedOutputTx(tx, localRevocationPubkey, localParams.toSelfDelay, localDelayedPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKw)
+        val claimDelayed = Transactions.makeClaimDelayedOutputTx(tx, Satoshi(localParams.dustLimitSatoshis), localRevocationPubkey, localParams.toSelfDelay, localDelayedPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKwDelayed)
         val sig = Transactions.sign(claimDelayed, localDelayedPrivkey)
         Transactions.addSigs(claimDelayed, sig)
       })
@@ -243,7 +255,7 @@ object Helpers {
       val htlcDelayedTxes = htlcTxes.map {
         case txinfo: TransactionWithInputInfo => generateTx("claim-delayed-output")(Try {
           // TODO: we should use the current fee rate, not the initial fee rate that we get from localParams
-          val claimDelayed = Transactions.makeClaimDelayedOutputTx(txinfo.tx, localRevocationPubkey, localParams.toSelfDelay, localDelayedPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKw)
+          val claimDelayed = Transactions.makeClaimDelayedOutputTx(txinfo.tx, Satoshi(localParams.dustLimitSatoshis), localRevocationPubkey, localParams.toSelfDelay, localDelayedPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKwDelayed)
           val sig = Transactions.sign(claimDelayed, localDelayedPrivkey)
           Transactions.addSigs(claimDelayed, sig)
         })
@@ -258,7 +270,8 @@ object Helpers {
         claimMainDelayedOutputTx = mainDelayedTx.map(_.tx),
         htlcSuccessTxs = htlcTxes.collect { case c: HtlcSuccessTx => c.tx },
         htlcTimeoutTxs = htlcTxes.collect { case c: HtlcTimeoutTx => c.tx },
-        claimHtlcDelayedTx = htlcDelayedTxes.map(_.tx))
+        claimHtlcDelayedTx = htlcDelayedTxes.map(_.tx),
+        spent = Map.empty)
     }
 
     /**
@@ -274,20 +287,23 @@ object Helpers {
       val (remoteCommitTx, htlcTimeoutTxs, htlcSuccessTxs) = Commitments.makeRemoteTxs(remoteCommit.index, localParams, remoteParams, commitInput, remoteCommit.remotePerCommitmentPoint, remoteCommit.spec)
       require(remoteCommitTx.tx.txid == tx.txid, "txid mismatch, cannot recompute the current remote commit tx")
 
-      val remotePubkey = Generators.derivePubKey(remoteParams.paymentBasepoint, remoteCommit.remotePerCommitmentPoint)
-      val localPrivkey = Generators.derivePrivKey(localParams.paymentKey, remoteCommit.remotePerCommitmentPoint)
+      val localPaymentPrivkey = Generators.derivePrivKey(localParams.paymentKey, remoteCommit.remotePerCommitmentPoint)
+      val localHtlcPrivkey = Generators.derivePrivKey(localParams.htlcKey, remoteCommit.remotePerCommitmentPoint)
+      val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, remoteCommit.remotePerCommitmentPoint)
       val localPerCommitmentPoint = Generators.perCommitPoint(localParams.shaSeed, commitments.localCommit.index.toInt)
       val localRevocationPubKey = Generators.revocationPubKey(remoteParams.revocationBasepoint, localPerCommitmentPoint)
       val remoteRevocationPubkey = Generators.revocationPubKey(localParams.revocationBasepoint, remoteCommit.remotePerCommitmentPoint)
 
-      // for now we use the same fee rate they used, it should be up-to-date
-      val feeratePerKw = remoteCommit.spec.feeratePerKw
+      // no need to use a high fee rate for our main output (we are the only one who can spend it)
+      val feeratePerKwMain = Globals.feeratesPerKw.get.blocks_6
+      // we need to use a rather high fee for htlc-claim because we compete with the counterparty
+      val feeratePerKwHtlc = Globals.feeratesPerKw.get.block_1
 
       // first we will claim our main output right away
       val mainTx = generateTx("claim-p2wpkh-output")(Try {
-        val claimMain = Transactions.makeClaimP2WPKHOutputTx(tx, localPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKw)
-        val sig = Transactions.sign(claimMain, localPrivkey)
-        Transactions.addSigs(claimMain, localPrivkey.publicKey, sig)
+        val claimMain = Transactions.makeClaimP2WPKHOutputTx(tx, Satoshi(localParams.dustLimitSatoshis), localPaymentPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKwMain)
+        val sig = Transactions.sign(claimMain, localPaymentPrivkey)
+        Transactions.addSigs(claimMain, localPaymentPrivkey.publicKey, sig)
       })
 
       // those are the preimages to existing received htlcs
@@ -296,19 +312,19 @@ object Helpers {
       // remember we are looking at the remote commitment so IN for them is really OUT for us and vice versa
       val txes = commitments.remoteCommit.spec.htlcs.collect {
         // incoming htlc for which we have the preimage: we spend it directly
-        case Htlc(OUT, add: UpdateAddHtlc, _) if preimages.exists(r => sha256(r) == add.paymentHash) => generateTx("claim-htlc-success")(Try {
+        case DirectedHtlc(OUT, add: UpdateAddHtlc) if preimages.exists(r => sha256(r) == add.paymentHash) => generateTx("claim-htlc-success")(Try {
           val preimage = preimages.find(r => sha256(r) == add.paymentHash).get
-          val tx = Transactions.makeClaimHtlcSuccessTx(remoteCommitTx.tx, localPrivkey.publicKey, remotePubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKw)
-          val sig = Transactions.sign(tx, localPrivkey)
+          val tx = Transactions.makeClaimHtlcSuccessTx(remoteCommitTx.tx, Satoshi(localParams.dustLimitSatoshis), localHtlcPrivkey.publicKey, remoteHtlcPubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc)
+          val sig = Transactions.sign(tx, localHtlcPrivkey)
           Transactions.addSigs(tx, sig, preimage)
         })
 
         // (incoming htlc for which we don't have the preimage: nothing to do, it will timeout eventually and they will get their funds back)
 
         // outgoing htlc: they may or may not have the preimage, the only thing to do is try to get back our funds after timeout
-        case Htlc(IN, add: UpdateAddHtlc, _) => generateTx("claim-htlc-timeout")(Try {
-          val tx = Transactions.makeClaimHtlcTimeoutTx(remoteCommitTx.tx, localPrivkey.publicKey, remotePubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKw)
-          val sig = Transactions.sign(tx, localPrivkey)
+        case DirectedHtlc(IN, add: UpdateAddHtlc) => generateTx("claim-htlc-timeout")(Try {
+          val tx = Transactions.makeClaimHtlcTimeoutTx(remoteCommitTx.tx, Satoshi(localParams.dustLimitSatoshis), localHtlcPrivkey.publicKey, remoteHtlcPubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc)
+          val sig = Transactions.sign(tx, localHtlcPrivkey)
           Transactions.addSigs(tx, sig)
         })
       }.toSeq.flatten
@@ -320,7 +336,8 @@ object Helpers {
         commitTx = tx,
         claimMainOutputTx = mainTx.map(_.tx),
         claimHtlcSuccessTxs = txes.toList.collect { case c: ClaimHtlcSuccessTx => c.tx },
-        claimHtlcTimeoutTxs = txes.toList.collect { case c: ClaimHtlcTimeoutTx => c.tx }
+        claimHtlcTimeoutTxs = txes.toList.collect { case c: ClaimHtlcTimeoutTx => c.tx },
+        spent = Map.empty
       )
 
     }
@@ -348,16 +365,18 @@ object Helpers {
         .map { remotePerCommitmentSecret =>
           val remotePerCommitmentPoint = remotePerCommitmentSecret.toPoint
 
-          val remoteDelayedPubkey = Generators.derivePubKey(remoteParams.delayedPaymentBasepoint, remotePerCommitmentPoint)
+          val remoteDelayedPaymentPubkey = Generators.derivePubKey(remoteParams.delayedPaymentBasepoint, remotePerCommitmentPoint)
           val remoteRevocationPrivkey = Generators.revocationPrivKey(localParams.revocationSecret, remotePerCommitmentSecret)
           val localPrivkey = Generators.derivePrivKey(localParams.paymentKey, remotePerCommitmentPoint)
 
-          // for now we use the current commit's fee rate, it should be up-to-date
-          val feeratePerKw = localCommit.spec.feeratePerKw
+          // no need to use a high fee rate for our main output (we are the only one who can spend it)
+          val feeratePerKwMain = Globals.feeratesPerKw.get.blocks_6
+          // we need to use a high fee here for punishment txes because after a delay they can be spent by the counterparty
+          val feeratePerKwPenalty = Globals.feeratesPerKw.get.block_1
 
           // first we will claim our main output right away
           val mainTx = generateTx("claim-p2wpkh-output")(Try {
-            val claimMain = Transactions.makeClaimP2WPKHOutputTx(tx, localPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKw)
+            val claimMain = Transactions.makeClaimP2WPKHOutputTx(tx, Satoshi(localParams.dustLimitSatoshis), localPrivkey.publicKey, localParams.defaultFinalScriptPubKey, feeratePerKwMain)
             val sig = Transactions.sign(claimMain, localPrivkey)
             Transactions.addSigs(claimMain, localPrivkey.publicKey, sig)
           })
@@ -365,7 +384,7 @@ object Helpers {
           // then we punish them by stealing their main output
           val mainPenaltyTx = generateTx("main-penalty")(Try {
             // TODO: we should use the current fee rate, not the initial fee rate that we get from localParams
-            val txinfo = Transactions.makeMainPenaltyTx(tx, remoteRevocationPrivkey.publicKey, localParams.defaultFinalScriptPubKey, remoteParams.toSelfDelay, remoteDelayedPubkey, feeratePerKw)
+            val txinfo = Transactions.makeMainPenaltyTx(tx, Satoshi(localParams.dustLimitSatoshis), remoteRevocationPrivkey.publicKey, localParams.defaultFinalScriptPubKey, remoteParams.toSelfDelay, remoteDelayedPaymentPubkey, feeratePerKwPenalty)
             val sig = Transactions.sign(txinfo, remoteRevocationPrivkey)
             Transactions.addSigs(txinfo, sig)
           })
@@ -382,11 +401,145 @@ object Helpers {
             mainPenaltyTx = mainPenaltyTx.map(_.tx),
             claimHtlcTimeoutTxs = Nil,
             htlcTimeoutTxs = Nil,
-            htlcPenaltyTxs = Nil
+            htlcPenaltyTxs = Nil,
+            spent = Map.empty
           )
         }
     }
 
+    /**
+      * In CLOSING state, when we are notified that a transaction has been confirmed, we check if this tx belongs in the
+      * local commit scenario and keep track of it.
+      *
+      * We need to keep track of all transactions spending the outputs of the commitment tx, because some outputs can be
+      * spent both by us and our counterparty. Because of that, some of our transactions may never confirm and we don't
+      * want to wait forever before declaring that the channel is CLOSED.
+      *
+      * @param localCommitPublished
+      * @return
+      */
+    def updateLocalCommitPublished(localCommitPublished: LocalCommitPublished, tx: Transaction) = {
+      // even if our txes only have one input, maybe our counterparty uses a different scheme so we need to iterate
+      // over all of them to check if they are relevant
+      val relevantOutpoints = tx.txIn.map(_.outPoint).filter { outPoint =>
+        // is this the commit tx itself ? (we could do this outside of the loop...)
+        val isCommitTx = localCommitPublished.commitTx.txid == tx.txid
+        // does the tx spend an output of the local commitment tx?
+        val spendsTheCommitTx = localCommitPublished.commitTx.txid == outPoint.txid
+        // is the tx one of our 3rd stage delayed txes? (a 3rd stage tx is a tx spending the output of an htlc tx, which
+        // is itself spending the output of the commitment tx)
+        val is3rdStageDelayedTx = localCommitPublished.claimHtlcDelayedTx.map(_.txid).contains(outPoint.txid)
+        isCommitTx || spendsTheCommitTx || is3rdStageDelayedTx
+      }
+      // then we add the relevant outpoints to the map keeping track of which txid spends which outpoint
+      localCommitPublished.copy(spent = localCommitPublished.spent ++ relevantOutpoints.map(o => (o -> tx.txid)).toMap)
+    }
+
+    /**
+      * In CLOSING state, when we are notified that a transaction has been confirmed, we check if this tx belongs in the
+      * remote commit scenario and keep track of it.
+      *
+      * We need to keep track of all transactions spending the outputs of the commitment tx, because some outputs can be
+      * spent both by us and our counterparty. Because of that, some of our transactions may never confirm and we don't
+      * want to wait forever before declaring that the channel is CLOSED.
+      *
+      * @param remoteCommitPublished
+      * @return
+      */
+    def updateRemoteCommitPublished(remoteCommitPublished: RemoteCommitPublished, tx: Transaction) = {
+      // even if our txes only have one input, maybe our counterparty uses a different scheme so we need to iterate
+      // over all of them to check if they are relevant
+      val relevantOutpoints = tx.txIn.map(_.outPoint).filter { outPoint =>
+        // is this the commit tx itself ? (we could do this outside of the loop...)
+        val isCommitTx = remoteCommitPublished.commitTx.txid == tx.txid
+        // does the tx spend an output of the local commitment tx?
+        val spendsTheCommitTx = remoteCommitPublished.commitTx.txid == outPoint.txid
+        // TODO: we don't currently spend htlc transactions
+        isCommitTx || spendsTheCommitTx
+      }
+      // then we add the relevant outpoints to the map keeping track of which txid spends which outpoint
+      remoteCommitPublished.copy(spent = remoteCommitPublished.spent ++ relevantOutpoints.map(o => (o -> tx.txid)).toMap)
+    }
+
+    /**
+      * In CLOSING state, when we are notified that a transaction has been confirmed, we check if this tx belongs in the
+      * revoked commit scenario and keep track of it.
+      *
+      * We need to keep track of all transactions spending the outputs of the commitment tx, because some outputs can be
+      * spent both by us and our counterparty. Because of that, some of our transactions may never confirm and we don't
+      * want to wait forever before declaring that the channel is CLOSED.
+      *
+      * @param revokedCommitPublished
+      * @return
+      */
+    def updateRevokedCommitPublished(revokedCommitPublished: RevokedCommitPublished, tx: Transaction) = {
+      // even if our txes only have one input, maybe our counterparty uses a different scheme so we need to iterate
+      // over all of them to check if they are relevant
+      val relevantOutpoints = tx.txIn.map(_.outPoint).filter { outPoint =>
+        // is this the commit tx itself ? (we could do this outside of the loop...)
+        val isCommitTx = revokedCommitPublished.commitTx.txid == tx.txid
+        // does the tx spend an output of the local commitment tx?
+        val spendsTheCommitTx = revokedCommitPublished.commitTx.txid == outPoint.txid
+        isCommitTx || spendsTheCommitTx
+      }
+      // then we add the relevant outpoints to the map keeping track of which txid spends which outpoint
+      revokedCommitPublished.copy(spent = revokedCommitPublished.spent ++ relevantOutpoints.map(o => (o -> tx.txid)).toMap)
+    }
+
+    /**
+      * A local commit is considered done when:
+      * - all commitment tx outputs that we can spend have been spent and confirmed (even if the spending tx was not ours)
+      * - all 3rd stage txes (txes spending htlc txes) have been confirmed
+      *
+      * @param localCommitPublished
+      * @return
+      */
+    def isLocalCommitDone(localCommitPublished: LocalCommitPublished) = {
+      // is the commitment tx buried? (we need to check this because we may not have nay outputs)
+      val isCommitTxConfirmed = localCommitPublished.spent.values.toSet.contains(localCommitPublished.commitTx.txid)
+      // are there remaining spendable outputs from the commitment tx? we just substract all known spent outputs from the ones we control
+      val commitOutputsSpendableByUs = (localCommitPublished.claimMainDelayedOutputTx.toSeq ++ localCommitPublished.htlcSuccessTxs ++ localCommitPublished.htlcTimeoutTxs)
+        .flatMap(_.txIn.map(_.outPoint)).toSet -- localCommitPublished.spent.keys
+      // which htlc delayed txes can we expect to be confirmed?
+      val unconfirmedHtlcDelayedTxes = localCommitPublished.claimHtlcDelayedTx
+        .filter(tx => (tx.txIn.map(_.outPoint.txid).toSet -- localCommitPublished.spent.values).isEmpty) // only the txes which parents are already confirmed may get confirmed (note that this also eliminates outputs that have been double-spent by a competing tx)
+        .filterNot(tx => localCommitPublished.spent.values.toSet.contains(tx.txid)) // has the tx already been confirmed?
+      isCommitTxConfirmed && commitOutputsSpendableByUs.isEmpty && unconfirmedHtlcDelayedTxes.isEmpty
+    }
+
+    /**
+      * A remote commit is considered done when all commitment tx outputs that we can spend have been spent and confirmed
+      * (even if the spending tx was not ours).
+      *
+      * @param remoteCommitPublished
+      * @return
+      */
+    def isRemoteCommitDone(remoteCommitPublished: RemoteCommitPublished) = {
+      // is the commitment tx buried? (we need to check this because we may not have nay outputs)
+      val isCommitTxConfirmed = remoteCommitPublished.spent.values.toSet.contains(remoteCommitPublished.commitTx.txid)
+      // are there remaining spendable outputs from the commitment tx?
+      val commitOutputsSpendableByUs = (remoteCommitPublished.claimMainOutputTx.toSeq ++ remoteCommitPublished.claimHtlcSuccessTxs ++ remoteCommitPublished.claimHtlcTimeoutTxs)
+        .flatMap(_.txIn.map(_.outPoint)).toSet -- remoteCommitPublished.spent.keys
+      isCommitTxConfirmed && commitOutputsSpendableByUs.isEmpty
+    }
+
+    /**
+      * A remote commit is considered done when all commitment tx outputs that we can spend have been spent and confirmed
+      * (even if the spending tx was not ours).
+      *
+      * @param revokedCommitPublished
+      * @return
+      */
+    def isRevokedCommitDone(revokedCommitPublished: RevokedCommitPublished) = {
+      // is the commitment tx buried? (we need to check this because we may not have nay outputs)
+      val isCommitTxConfirmed = revokedCommitPublished.spent.values.toSet.contains(revokedCommitPublished.commitTx.txid)
+      // are there remaining spendable outputs from the commitment tx?
+      val commitOutputsSpendableByUs = (revokedCommitPublished.claimMainOutputTx.toSeq ++ revokedCommitPublished.mainPenaltyTx)
+        .flatMap(_.txIn.map(_.outPoint)).toSet -- revokedCommitPublished.spent.keys
+      // TODO: we don't currently spend htlc transactions
+      isCommitTxConfirmed && commitOutputsSpendableByUs.isEmpty
+    }
+
   }
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/channel/Register.scala

@@ -3,7 +3,7 @@ package fr.acinq.eclair.channel
 import akka.actor.Status.Failure
 import akka.actor.{Actor, ActorLogging, ActorRef, Terminated}
 import fr.acinq.bitcoin.BinaryData
-import fr.acinq.eclair.channel.Register.{Forward, ForwardShortId}
+import fr.acinq.eclair.channel.Register.{Forward, ForwardFailure, ForwardShortId, ForwardShortIdFailure}
 
 /**
   * Created by PM on 26/01/2016.
@@ -42,23 +42,27 @@ class Register extends Actor with ActorLogging {
 
     case 'shortIds => sender ! shortIds
 
-    case Forward(channelId, msg) =>
+    case fwd@Forward(channelId, msg) =>
       channels.get(channelId) match {
         case Some(channel) => channel forward msg
-        case None => sender ! Failure(new RuntimeException(s"channel $channelId not found"))
+        case None => sender ! Failure(ForwardFailure(fwd))
       }
 
-    case ForwardShortId(shortChannelId, msg) =>
+    case fwd@ForwardShortId(shortChannelId, msg) =>
       shortIds.get(shortChannelId).flatMap(channels.get(_)) match {
         case Some(channel) => channel forward msg
-        case None => sender ! Failure(new RuntimeException(s"channel $shortChannelId not found"))
+        case None => sender ! Failure(ForwardShortIdFailure(fwd))
       }
   }
 }
 
 object Register {
+
   // @formatter:off
-  case class Forward(channelId: BinaryData, message: Any)
-  case class ForwardShortId(shortChannelId: Long, message: Any)
+  case class Forward[T](channelId: BinaryData, message: T)
+  case class ForwardShortId[T](shortChannelId: Long, message: T)
+
+  case class ForwardFailure[T](fwd: Forward[T]) extends RuntimeException(s"channel ${fwd.channelId} not found")
+  case class ForwardShortIdFailure[T](fwd: ForwardShortId[T]) extends RuntimeException(s"channel ${fwd.shortChannelId} not found")
   // @formatter:on
 }

eclair-core/src/main/scala/fr/acinq/eclair/crypto/BitStream.scala

@@ -1,15 +1,15 @@
 package fr.acinq.eclair.crypto
 
-import fr.acinq.bitcoin.BinaryData
 import org.spongycastle.util.encoders.Hex
 
 import scala.annotation.tailrec
 
 /**
   * Bit stream that can be written to and read at both ends (i.e. you can read from the end or the beginning of the stream)
-  * @param bytes bits packed as bytes, the last byte is padded with 0s
+  *
+  * @param bytes    bits packed as bytes, the last byte is padded with 0s
   * @param offstart offset at which the first bit is in the first byte
-  * @param offend offset at which the last bit is in the last byte
+  * @param offend   offset at which the last bit is in the last byte
   */
 case class BitStream(bytes: Vector[Byte], offstart: Int, offend: Int) {
 
@@ -20,6 +20,7 @@ case class BitStream(bytes: Vector[Byte], offstart: Int, offend: Int) {
   def bitCount = 8 * bytes.length - offstart - offend
 
   def isEmpty = bitCount == 0
+
   /**
     * append a byte to a bitstream
     *
@@ -97,16 +98,17 @@ case class BitStream(bytes: Vector[Byte], offstart: Int, offend: Int) {
       BitStream(bytes.dropRight(2) :+ a1.toByte, offstart, offend) -> byte.toByte
   }
 
-  def popBytes(n: Int) : (BitStream, Seq[Byte]) = {
+  def popBytes(n: Int): (BitStream, Seq[Byte]) = {
     @tailrec
-    def loop(stream: BitStream, acc: Seq[Byte]) : (BitStream, Seq[Byte]) =
+    def loop(stream: BitStream, acc: Seq[Byte]): (BitStream, Seq[Byte]) =
       if (acc.length == n) (stream, acc) else {
-      val (stream1, value) = stream.popByte
-      loop(stream1, acc :+ value)
-    }
+        val (stream1, value) = stream.popByte
+        loop(stream1, acc :+ value)
+      }
 
     loop(this, Nil)
   }
+
   /**
     * read the first bit from a bitstream
     *
@@ -117,7 +119,7 @@ case class BitStream(bytes: Vector[Byte], offstart: Int, offend: Int) {
     case _ => BitStream(bytes, offstart + 1, offend) -> firstBit
   }
 
-  def readBits(count: Int) : (BitStream, Seq[Bit]) = {
+  def readBits(count: Int): (BitStream, Seq[Bit]) = {
     @tailrec
     def loop(stream: BitStream, acc: Seq[Bit]): (BitStream, Seq[Bit]) = if (acc.length == count) (stream, acc) else {
       val (stream1, bit) = stream.readBit
@@ -126,14 +128,15 @@ case class BitStream(bytes: Vector[Byte], offstart: Int, offend: Int) {
 
     loop(this, Nil)
   }
+
   /**
     * read the first byte from a bitstream
     *
     * @return
     */
   def readByte: (BitStream, Byte) = {
-      val byte = ((bytes(0) << offstart) | (bytes(1) >>> (7 - offstart))) & 0xff
-      BitStream(bytes.tail, offstart, offend) -> byte.toByte
+    val byte = ((bytes(0) << offstart) | (bytes(1) >>> (7 - offstart))) & 0xff
+    BitStream(bytes.tail, offstart, offend) -> byte.toByte
   }
 
   def isSet(pos: Int): Boolean = {

eclair-core/src/main/scala/fr/acinq/eclair/crypto/Noise.scala

@@ -3,8 +3,8 @@ package fr.acinq.eclair.crypto
 import java.math.BigInteger
 import java.nio.ByteOrder
 
-import fr.acinq.eclair.randomBytes
 import fr.acinq.bitcoin.{BinaryData, Crypto, Protocol}
+import fr.acinq.eclair.randomBytes
 import grizzled.slf4j.Logging
 import org.spongycastle.crypto.digests.SHA256Digest
 import org.spongycastle.crypto.macs.HMac

eclair-core/src/main/scala/fr/acinq/eclair/crypto/ShaChain.scala

@@ -97,7 +97,7 @@ object ShaChain {
   }
 
 
-  val shaChainCodec: Codec[ShaChain] =  {
+  val shaChainCodec: Codec[ShaChain] = {
     import scodec.Codec
     import scodec.bits.BitVector
     import scodec.codecs._
@@ -106,7 +106,7 @@ object ShaChain {
     val entryCodec = vectorOfN(uint16, bool) ~ LightningMessageCodecs.varsizebinarydata
 
     // codec for a Map[Vector[Boolean], BinaryData]: write all k ->v pairs using the codec defined above
-    val mapCodec: Codec[Map[Vector[Boolean], BinaryData]] = Codec[Map[Vector[Boolean], BinaryData]] (
+    val mapCodec: Codec[Map[Vector[Boolean], BinaryData]] = Codec[Map[Vector[Boolean], BinaryData]](
       (m: Map[Vector[Boolean], BinaryData]) => vectorOfN(uint16, entryCodec).encode(m.toVector),
       (b: BitVector) => vectorOfN(uint16, entryCodec).decode(b).map(_.map(_.toMap))
     )

eclair-core/src/main/scala/fr/acinq/eclair/crypto/Sphinx.scala

@@ -5,7 +5,7 @@ import java.nio.ByteOrder
 
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.bitcoin.{BinaryData, Crypto, Protocol}
-import fr.acinq.eclair.wire.{ChannelUpdate, FailureMessage, FailureMessageCodecs, LightningMessageCodecs}
+import fr.acinq.eclair.wire.{FailureMessage, FailureMessageCodecs}
 import grizzled.slf4j.Logging
 import org.spongycastle.crypto.digests.SHA256Digest
 import org.spongycastle.crypto.macs.HMac
@@ -32,7 +32,7 @@ object Sphinx extends Logging {
 
   // onion packet length
   val PacketLength = 1 + 33 + MacLength + MaxHops * (PayloadLength + MacLength)
-  
+
   // last packet (all zeroes except for the version byte)
   val LAST_PACKET = Packet(Version, zeroes(33), zeroes(MacLength), zeroes(MaxHops * (PayloadLength + MacLength)))
 

eclair-core/src/main/scala/fr/acinq/eclair/crypto/TransportHandler.scala

@@ -2,12 +2,13 @@ package fr.acinq.eclair.crypto
 
 import java.nio.ByteOrder
 
-import akka.actor.{Actor, ActorRef, FSM, Terminated}
+import akka.actor.{Actor, ActorRef, FSM, Props, Terminated}
 import akka.io.Tcp.{PeerClosed, _}
 import akka.util.ByteString
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{BinaryData, Protocol}
 import fr.acinq.eclair.crypto.Noise._
+import fr.acinq.eclair.io.WriteAckSender
 import scodec.bits.BitVector
 import scodec.{Attempt, Codec, DecodeResult}
 
@@ -33,6 +34,10 @@ class TransportHandler[T: ClassTag](keyPair: KeyPair, rs: Option[BinaryData], co
 
   connection ! akka.io.Tcp.Register(self)
 
+  val out = context.actorOf(Props(new WriteAckSender(connection)))
+
+  def buf(message: BinaryData): ByteString = ByteString.fromArray(message)
+
   // it means we initiate the dialog
   val isWriter = rs.isDefined
 
@@ -42,7 +47,7 @@ class TransportHandler[T: ClassTag](keyPair: KeyPair, rs: Option[BinaryData], co
     val state = makeWriter(keyPair, rs.get)
     val (state1, message, None) = state.write(BinaryData.empty)
     log.debug(s"sending prefix + $message")
-    connection ! Write(TransportHandler.prefix +: message)
+    out ! buf(TransportHandler.prefix +: message)
     state1
   } else {
     makeReader(keyPair)
@@ -82,11 +87,11 @@ class TransportHandler[T: ClassTag](keyPair: KeyPair, rs: Option[BinaryData], co
                 // we're still in the middle of the handshake process and the other end must first received our next
                 // message before they can reply
                 require(remainder.isEmpty, "unexpected additional data received during handshake")
-                connection ! Write(TransportHandler.prefix +: message)
+                out ! buf(TransportHandler.prefix +: message)
                 stay using HandshakeData(reader1, remainder)
               }
               case (_, message, Some((enc, dec, ck))) => {
-                connection ! Write(TransportHandler.prefix +: message)
+                out ! buf(TransportHandler.prefix +: message)
                 val remoteNodeId = PublicKey(writer.rs)
                 context.parent ! HandshakeCompleted(self, remoteNodeId)
                 val nextStateData = WaitingForListenerData(ExtendedCipherState(enc, ck), ExtendedCipherState(dec, ck), remainder)
@@ -120,7 +125,7 @@ class TransportHandler[T: ClassTag](keyPair: KeyPair, rs: Option[BinaryData], co
     case Event(t: T, WaitingForCyphertextData(enc, dec, length, buffer, listener)) =>
       val blob = codec.encode(t).require.toByteArray
       val (enc1, ciphertext) = TransportHandler.encrypt(enc, blob)
-      connection ! Write(ByteString.fromArray(ciphertext.toArray))
+      out ! buf(ciphertext)
       stay using WaitingForCyphertextData(enc1, dec, length, buffer, listener)
   }
 

eclair-core/src/main/scala/fr/acinq/eclair/db/NetworkDb.scala

@@ -17,6 +17,7 @@ trait NetworkDb {
 
   /**
     * This method removes 1 channel announcement and 2 channel updates (at both ends of the same channel)
+    *
     * @param shortChannelId
     * @return
     */

eclair-core/src/main/scala/fr/acinq/eclair/db/PreimagesDb.scala

@@ -0,0 +1,25 @@
+package fr.acinq.eclair.db
+
+import fr.acinq.bitcoin.BinaryData
+
+/**
+  * This database stores the preimages that we have received from downstream
+  * (either directly via UpdateFulfillHtlc or by extracting the value from the
+  * blockchain).
+  *
+  * This means that this database is only used in the context of *relaying* payments.
+  *
+  * We need to be sure that if downstream is able to pulls funds from us, we can always
+  * do the same from upstream, otherwise we lose money. Hence the need for persistence
+  * to handle all corner cases.
+  *
+  */
+trait PreimagesDb {
+
+  def addPreimage(channelId: BinaryData, htlcId: Long, paymentPreimage: BinaryData)
+
+  def removePreimage(channelId: BinaryData, htlcId: Long)
+
+  def listPreimages(channelId: BinaryData): List[(BinaryData, Long, BinaryData)]
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteChannelsDb.scala

@@ -30,9 +30,13 @@ class SqliteChannelsDb(sqlite: Connection) extends ChannelsDb {
   }
 
   override def removeChannel(channelId: BinaryData): Unit = {
-    val statement = sqlite.prepareStatement("DELETE FROM local_channels WHERE channel_id=?")
-    statement.setBytes(1, channelId)
-    statement.executeUpdate()
+    val statement1 = sqlite.prepareStatement("DELETE FROM preimages WHERE channel_id=?")
+    statement1.setBytes(1, channelId)
+    statement1.executeUpdate()
+
+    val statement2 = sqlite.prepareStatement("DELETE FROM local_channels WHERE channel_id=?")
+    statement2.setBytes(1, channelId)
+    statement2.executeUpdate()
   }
 
   override def listChannels(): List[HasCommitments] = {

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteNetworkDb.scala

@@ -1,14 +1,12 @@
 package fr.acinq.eclair.db.sqlite
 
-import java.sql.{Connection, ResultSet}
+import java.sql.Connection
 
 import fr.acinq.bitcoin.Crypto
 import fr.acinq.eclair.db.NetworkDb
 import fr.acinq.eclair.router.Announcements
 import fr.acinq.eclair.wire.LightningMessageCodecs.{channelAnnouncementCodec, channelUpdateCodec, nodeAnnouncementCodec}
 import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement}
-import scodec.Codec
-import scodec.bits.BitVector
 
 class SqliteNetworkDb(sqlite: Connection) extends NetworkDb {
 
@@ -19,7 +17,7 @@ class SqliteNetworkDb(sqlite: Connection) extends NetworkDb {
     statement.execute("PRAGMA foreign_keys = ON")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS nodes (node_id BLOB NOT NULL PRIMARY KEY, data BLOB NOT NULL)")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS channels (short_channel_id INTEGER NOT NULL PRIMARY KEY, data BLOB NOT NULL)")
-    statement.executeUpdate("CREATE TABLE IF NOT EXISTS channel_updates (short_channel_id INTEGER NOT NULL, node_flag INTEGER NOT NULL, data BLOB NOT NULL, FOREIGN KEY(short_channel_id) REFERENCES channels(short_channel_id))")
+    statement.executeUpdate("CREATE TABLE IF NOT EXISTS channel_updates (short_channel_id INTEGER NOT NULL, node_flag INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY(short_channel_id, node_flag), FOREIGN KEY(short_channel_id) REFERENCES channels(short_channel_id))")
     statement.executeUpdate("CREATE INDEX IF NOT EXISTS channel_updates_idx ON channel_updates(short_channel_id)")
   }
 

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePreimagesDb.scala

@@ -0,0 +1,41 @@
+package fr.acinq.eclair.db.sqlite
+
+import java.sql.Connection
+
+import fr.acinq.bitcoin.BinaryData
+import fr.acinq.eclair.db.PreimagesDb
+
+class SqlitePreimagesDb(sqlite: Connection) extends PreimagesDb {
+
+  {
+    val statement = sqlite.createStatement
+    // note: should we use a foreign key to local_channels table here?
+    statement.executeUpdate("CREATE TABLE IF NOT EXISTS preimages (channel_id BLOB NOT NULL, htlc_id INTEGER NOT NULL, preimage BLOB NOT NULL, PRIMARY KEY(channel_id, htlc_id))")
+  }
+
+  override def addPreimage(channelId: BinaryData, htlcId: Long, paymentPreimage: BinaryData): Unit = {
+    val statement = sqlite.prepareStatement("INSERT OR IGNORE INTO preimages VALUES (?, ?, ?)")
+    statement.setBytes(1, channelId)
+    statement.setLong(2, htlcId)
+    statement.setBytes(3, paymentPreimage)
+    statement.executeUpdate()
+  }
+
+  override def removePreimage(channelId: BinaryData, htlcId: Long): Unit = {
+    val statement = sqlite.prepareStatement("DELETE FROM preimages WHERE channel_id=? AND htlc_id=?")
+    statement.setBytes(1, channelId)
+    statement.setLong(2, htlcId)
+    statement.executeUpdate()
+  }
+
+  override def listPreimages(channelId: BinaryData): List[(BinaryData, Long, BinaryData)] = {
+    val statement = sqlite.prepareStatement("SELECT htlc_id, preimage FROM preimages WHERE channel_id=?")
+    statement.setBytes(1, channelId)
+    val rs = statement.executeQuery()
+    var l: List[(BinaryData, Long, BinaryData)] = Nil
+    while (rs.next()) {
+      l = l :+ (channelId, rs.getLong("htlc_id"), BinaryData(rs.getBytes("preimage")))
+    }
+    l
+  }
+}

eclair-core/src/main/scala/fr/acinq/eclair/io/Peer.scala

@@ -6,7 +6,7 @@ import akka.actor.{ActorRef, LoggingFSM, OneForOneStrategy, PoisonPill, Props, S
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.bitcoin.{BinaryData, Crypto, DeterministicWallet}
 import fr.acinq.eclair._
-import fr.acinq.eclair.blockchain.wallet.EclairWallet
+import fr.acinq.eclair.blockchain.EclairWallet
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.TransportHandler.{HandshakeCompleted, Listener}
 import fr.acinq.eclair.io.Switchboard.{NewChannel, NewConnection}
@@ -178,7 +178,8 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, address_opt: Option[
       log.info(s"requesting a new channel to $remoteNodeId with fundingSatoshis=${c.fundingSatoshis} and pushMsat=${c.pushMsat}")
       val (channel, localParams) = createChannel(nodeParams, transport, funder = true, c.fundingSatoshis.toLong)
       val temporaryChannelId = randomBytes(32)
-      channel ! INPUT_INIT_FUNDER(temporaryChannelId, c.fundingSatoshis.amount, c.pushMsat.amount, Globals.feeratePerKw.get, localParams, transport, remoteInit, c.channelFlags.getOrElse(nodeParams.channelFlags))
+      val networkFeeratePerKw = Globals.feeratesPerKw.get.block_1
+      channel ! INPUT_INIT_FUNDER(temporaryChannelId, c.fundingSatoshis.amount, c.pushMsat.amount, networkFeeratePerKw, localParams, transport, remoteInit, c.channelFlags.getOrElse(nodeParams.channelFlags))
       stay using d.copy(channels = channels + (TemporaryChannelId(temporaryChannelId) -> channel))
 
     case Event(msg: OpenChannel, d@ConnectedData(transport, remoteInit, channels)) if !channels.contains(TemporaryChannelId(msg.temporaryChannelId)) =>
@@ -277,8 +278,9 @@ object Peer {
       revocationSecret = generateKey(nodeParams, keyIndex :: 1L :: Nil),
       paymentKey = generateKey(nodeParams, keyIndex :: 2L :: Nil),
       delayedPaymentKey = generateKey(nodeParams, keyIndex :: 3L :: Nil),
+      htlcKey = generateKey(nodeParams, keyIndex :: 4L :: Nil),
       defaultFinalScriptPubKey = defaultFinalScriptPubKey,
-      shaSeed = Crypto.sha256(generateKey(nodeParams, keyIndex :: 4L :: Nil).toBin), // TODO: check that
+      shaSeed = Crypto.sha256(generateKey(nodeParams, keyIndex :: 5L :: Nil).toBin), // TODO: check that
       isFunder = isFunder,
       globalFeatures = nodeParams.globalFeatures,
       localFeatures = nodeParams.localFeatures)

eclair-core/src/main/scala/fr/acinq/eclair/io/Server.scala

@@ -2,7 +2,7 @@ package fr.acinq.eclair.io
 
 import java.net.InetSocketAddress
 
-import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, Status, SupervisorStrategy, Terminated}
+import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, SupervisorStrategy}
 import akka.io.Tcp.SO.KeepAlive
 import akka.io.{IO, Tcp}
 import fr.acinq.eclair.NodeParams

eclair-core/src/main/scala/fr/acinq/eclair/io/Switchboard.scala

@@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, Stat
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{MilliSatoshi, Satoshi}
 import fr.acinq.eclair.NodeParams
-import fr.acinq.eclair.blockchain.wallet.EclairWallet
+import fr.acinq.eclair.blockchain.EclairWallet
 import fr.acinq.eclair.channel.HasCommitments
 import fr.acinq.eclair.crypto.TransportHandler.HandshakeCompleted
 import fr.acinq.eclair.router.Rebroadcast

eclair-core/src/main/scala/fr/acinq/eclair/io/WriteAckSender.scala

@@ -0,0 +1,48 @@
+package fr.acinq.eclair.io
+
+import akka.actor.{Actor, ActorLogging, ActorRef, PoisonPill}
+import akka.io.Tcp
+import akka.util.ByteString
+
+
+/**
+  * This implements an ACK-based throttling mechanism
+  * See https://doc.akka.io/docs/akka/snapshot/scala/io-tcp.html#throttling-reads-and-writes
+  */
+class WriteAckSender(connection: ActorRef) extends Actor with ActorLogging {
+
+  // Note: this actor should be killed if connection dies
+
+  case object Ack extends Tcp.Event
+
+  override def receive = idle
+
+  def idle: Receive = {
+    case data: ByteString =>
+      connection ! Tcp.Write(data, Ack)
+      context become buffering(Vector.empty[ByteString])
+  }
+
+  def buffering(buffer: Vector[ByteString]): Receive = {
+    case _: ByteString if buffer.size > MAX_BUFFERED =>
+      log.warning(s"buffer overrun, closing connection")
+      connection ! PoisonPill
+    case data: ByteString =>
+      log.debug(s"buffering write $data")
+      context become buffering(buffer :+ data)
+    case Ack =>
+      buffer.headOption match {
+        case Some(data) =>
+          connection ! Tcp.Write(data, Ack)
+          context become buffering(buffer.drop(1))
+        case None =>
+          log.debug(s"got last ack, back to idle")
+          context become idle
+      }
+  }
+
+  override def unhandled(message: Any): Unit = log.warning(s"unhandled message $message")
+
+  val MAX_BUFFERED = 100000L
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/package.scala

@@ -56,12 +56,12 @@ package object eclair {
   }
 
   /**
-    * Converts fee-rate-per-kB to fee-rate-per-kw, *based on a standard commit tx*
+    * Converts feerate in satoshi-per-bytes to feerate in satoshi-per-kw
     *
-    * @param feeratePerKb
-    * @return
+    * @param feeratePerByte feerate in satoshi-per-bytes
+    * @return feerate in satoshi-per-kw
     */
-  def feerateKb2Kw(feeratePerKb: Long): Long = feeratePerKb / 2
+  def feerateByte2Kw(feeratePerByte: Long): Long = feeratePerByte * 1024 / 4
 
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/payment/LocalPaymentHandler.scala

@@ -2,9 +2,9 @@ package fr.acinq.eclair.payment
 
 import akka.actor.{Actor, ActorLogging, Props, Status}
 import fr.acinq.bitcoin.{BinaryData, Crypto, MilliSatoshi}
-import fr.acinq.eclair.{Globals, NodeParams, randomBytes}
-import fr.acinq.eclair.channel.{CMD_FAIL_HTLC, CMD_FULFILL_HTLC, ExpiryTooSmall}
+import fr.acinq.eclair.channel.{CMD_FAIL_HTLC, CMD_FULFILL_HTLC}
 import fr.acinq.eclair.wire._
+import fr.acinq.eclair.{NodeParams, randomBytes}
 
 import scala.util.{Failure, Success, Try}
 
@@ -32,24 +32,25 @@ class LocalPaymentHandler(nodeParams: NodeParams) extends Actor with ActorLoggin
       }
 
     case htlc: UpdateAddHtlc =>
-     if (h2r.contains(htlc.paymentHash)) {
+      if (h2r.contains(htlc.paymentHash)) {
         val r = h2r(htlc.paymentHash)._1
         val pr = h2r(htlc.paymentHash)._2
         // The htlc amount must be equal or greater than the requested amount. A slight overpaying is permitted, however
         // it must not be greater than two times the requested amount.
         // see https://github.com/lightningnetwork/lightning-rfc/blob/master/04-onion-routing.md#failure-messages
-       pr.amount match {
-         case Some(amount) if MilliSatoshi(htlc.amountMsat) < amount => sender ! CMD_FAIL_HTLC(htlc.id, Right(IncorrectPaymentAmount), commit = true)
-         case Some(amount) if MilliSatoshi(htlc.amountMsat) > amount * 2 => sender ! CMD_FAIL_HTLC(htlc.id, Right(IncorrectPaymentAmount), commit = true)
-         case _ =>
-           // amount is correct or was not specified in the payment request
-           sender ! CMD_FULFILL_HTLC(htlc.id, r, commit = true)
-           context.system.eventStream.publish(PaymentReceived(MilliSatoshi(htlc.amountMsat), htlc.paymentHash))
-           context.become(run(h2r - htlc.paymentHash))
-       }
-     } else {
+        pr.amount match {
+          case Some(amount) if MilliSatoshi(htlc.amountMsat) < amount => sender ! CMD_FAIL_HTLC(htlc.id, Right(IncorrectPaymentAmount), commit = true)
+          case Some(amount) if MilliSatoshi(htlc.amountMsat) > amount * 2 => sender ! CMD_FAIL_HTLC(htlc.id, Right(IncorrectPaymentAmount), commit = true)
+          case _ =>
+            log.info(s"received payment for paymentHash=${htlc.paymentHash} amountMsat=${htlc.amountMsat}")
+            // amount is correct or was not specified in the payment request
+            sender ! CMD_FULFILL_HTLC(htlc.id, r, commit = true)
+            context.system.eventStream.publish(PaymentReceived(MilliSatoshi(htlc.amountMsat), htlc.paymentHash))
+            context.become(run(h2r - htlc.paymentHash))
+        }
+      } else {
         sender ! CMD_FAIL_HTLC(htlc.id, Right(UnknownPaymentHash), commit = true)
-     }
+      }
   }
 }
 

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentEvents.scala

@@ -11,6 +11,6 @@ sealed trait PaymentEvent {
 
 case class PaymentSent(amount: MilliSatoshi, feesPaid: MilliSatoshi, paymentHash: BinaryData) extends PaymentEvent
 
-case class PaymentRelayed(amount: MilliSatoshi, feesEarned: MilliSatoshi, paymentHash: BinaryData) extends PaymentEvent
+case class PaymentRelayed(amountIn: MilliSatoshi, amountOut: MilliSatoshi, paymentHash: BinaryData) extends PaymentEvent
 
 case class PaymentReceived(amount: MilliSatoshi, paymentHash: BinaryData) extends PaymentEvent

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentHop.scala

@@ -0,0 +1,46 @@
+package fr.acinq.eclair.payment
+
+import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.eclair.payment.PaymentRequest.ExtraHop
+import fr.acinq.eclair.wire.ChannelUpdate
+
+
+object PaymentHop {
+  /**
+    *
+    * @param baseMsat     fixed fee
+    * @param proportional proportional fee
+    * @param msat         amount in millisatoshi
+    * @return the fee (in msat) that a node should be paid to forward an HTLC of 'amount' millisatoshis
+    */
+  def nodeFee(baseMsat: Long, proportional: Long, msat: Long): Long = baseMsat + (proportional * msat) / 1000000
+
+  /**
+    *
+    * @param reversePath sequence of Hops from recipient to a start of assisted path
+    * @param msat        an amount to send to a payment recipient
+    * @return a sequence of extra hops with a pre-calculated fee for a given msat amount
+    */
+  def buildExtra(reversePath: Seq[Hop], msat: Long): Seq[ExtraHop] = (List.empty[ExtraHop] /: reversePath) {
+    case (Nil, hop) => ExtraHop(hop.nodeId, hop.shortChannelId, hop.nextFee(msat), hop.cltvExpiryDelta) :: Nil
+    case (head :: rest, hop) => ExtraHop(hop.nodeId, hop.shortChannelId, hop.nextFee(msat + head.fee), hop.cltvExpiryDelta) :: head :: rest
+  }
+}
+
+trait PaymentHop {
+  def nextFee(msat: Long): Long
+
+  def shortChannelId: Long
+
+  def cltvExpiryDelta: Int
+
+  def nodeId: PublicKey
+}
+
+case class Hop(nodeId: PublicKey, nextNodeId: PublicKey, lastUpdate: ChannelUpdate) extends PaymentHop {
+  def nextFee(msat: Long): Long = PaymentHop.nodeFee(lastUpdate.feeBaseMsat, lastUpdate.feeProportionalMillionths, msat)
+
+  def cltvExpiryDelta: Int = lastUpdate.cltvExpiryDelta
+
+  def shortChannelId: Long = lastUpdate.shortChannelId
+}

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentInitiator.scala

@@ -1,7 +1,6 @@
 package fr.acinq.eclair.payment
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
-import fr.acinq.bitcoin.BinaryData
 import fr.acinq.bitcoin.Crypto.PublicKey
 
 /**

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentLifecycle.scala

@@ -13,7 +13,7 @@ import scodec.Attempt
 
 // @formatter:off
 case class ReceivePayment(amountMsat: MilliSatoshi, description: String)
-case class SendPayment(amountMsat: Long, paymentHash: BinaryData, targetNodeId: PublicKey, maxAttempts: Int = 5)
+case class SendPayment(amountMsat: Long, paymentHash: BinaryData, targetNodeId: PublicKey, minFinalCltvExpiry: Long = PaymentLifecycle.defaultMinFinalCltvExpiry, maxAttempts: Int = 5)
 
 sealed trait PaymentResult
 case class PaymentSucceeded(route: Seq[Hop], paymentPreimage: BinaryData) extends PaymentResult
@@ -54,7 +54,7 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
     case Event(RouteResponse(hops, ignoreNodes, ignoreChannels), WaitingForRoute(s, c, failures)) =>
       log.info(s"route found: attempt=${failures.size + 1}/${c.maxAttempts} route=${hops.map(_.nextNodeId).mkString("->")} channels=${hops.map(_.lastUpdate.shortChannelId.toHexString).mkString("->")}")
       val firstHop = hops.head
-      val finalExpiry = Globals.blockCount.get().toInt + defaultHtlcExpiry
+      val finalExpiry = Globals.blockCount.get().toInt + c.minFinalCltvExpiry.toInt
       val (cmd, sharedSecrets) = buildCommand(c.amountMsat, finalExpiry, c.paymentHash, hops)
       // TODO: HACK!!!! see Router.scala (we actually store the first node id in the sig)
       if (firstHop.lastUpdate.signature.size == 32) {
@@ -132,6 +132,14 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
           goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ RemoteFailure(hops, e))
       }
 
+    case Event(fail: UpdateFailMalformedHtlc, _) =>
+      log.info(s"first node in the route couldn't parse our htlc: fail=$fail")
+      // this is a corner case, that can only happen when the *first* node in the route cannot parse the onion
+      // (if this happens higher up in the route, the error would be wrapped in an UpdateFailHtlc and handled above)
+      // let's consider it a local error and treat is as such
+      self ! Status.Failure(new RuntimeException("first hop returned an UpdateFailMalformedHtlc message"))
+      stay
+
     case Event(Status.Failure(t), WaitingForComplete(s, c, _, failures, _, ignoreNodes, ignoreChannels, hops)) =>
       if (failures.size + 1 >= c.maxAttempts) {
         s ! PaymentFailed(c.paymentHash, failures :+ LocalFailure(t))
@@ -151,15 +159,6 @@ object PaymentLifecycle {
 
   def props(sourceNodeId: PublicKey, router: ActorRef, register: ActorRef) = Props(classOf[PaymentLifecycle], sourceNodeId, router, register)
 
-  /**
-    *
-    * @param baseMsat     fixed fee
-    * @param proportional proportional fee
-    * @param msat         amount in millisatoshi
-    * @return the fee (in msat) that a node should be paid to forward an HTLC of 'amount' millisatoshis
-    */
-  def nodeFee(baseMsat: Long, proportional: Long, msat: Long): Long = baseMsat + (proportional * msat) / 1000000
-
   def buildOnion(nodes: Seq[PublicKey], payloads: Seq[PerHopPayload], associatedData: BinaryData): Sphinx.PacketAndSecrets = {
     require(nodes.size == payloads.size)
     val sessionKey = randomKey
@@ -176,22 +175,20 @@ object PaymentLifecycle {
     *
     * @param finalAmountMsat the final htlc amount in millisatoshis
     * @param finalExpiry     the final htlc expiry in number of blocks
-    * @param hops            the hops as computed by the router
+    * @param hops            the hops as computed by the router + extra routes from payment request
     * @return a (firstAmountMsat, firstExpiry, payloads) tuple where:
     *         - firstAmountMsat is the amount for the first htlc in the route
     *         - firstExpiry is the cltv expiry for the first htlc in the route
     *         - a sequence of payloads that will be used to build the onion
     */
-  def buildPayloads(finalAmountMsat: Long, finalExpiry: Int, hops: Seq[Hop]): (Long, Int, Seq[PerHopPayload]) =
+  def buildPayloads(finalAmountMsat: Long, finalExpiry: Int, hops: Seq[PaymentHop]): (Long, Int, Seq[PerHopPayload]) =
     hops.reverse.foldLeft((finalAmountMsat, finalExpiry, PerHopPayload(0L, finalAmountMsat, finalExpiry) :: Nil)) {
       case ((msat, expiry, payloads), hop) =>
-        val feeMsat = nodeFee(hop.lastUpdate.feeBaseMsat, hop.lastUpdate.feeProportionalMillionths, msat)
-        val expiryDelta = hop.lastUpdate.cltvExpiryDelta
-        (msat + feeMsat, expiry + expiryDelta, PerHopPayload(hop.lastUpdate.shortChannelId, msat, expiry) +: payloads)
+        (msat + hop.nextFee(msat), expiry + hop.cltvExpiryDelta, PerHopPayload(hop.shortChannelId, msat, expiry) +: payloads)
     }
 
-  // TODO: set correct initial expiry
-  val defaultHtlcExpiry = 10
+  // this is defined in BOLT 11
+  val defaultMinFinalCltvExpiry = 9
 
   def buildCommand(finalAmountMsat: Long, finalExpiry: Int, paymentHash: BinaryData, hops: Seq[Hop]): (CMD_ADD_HTLC, Seq[(BinaryData, PublicKey)]) = {
     val (firstAmountMsat, firstExpiry, payloads) = buildPayloads(finalAmountMsat, finalExpiry, hops.drop(1))

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentRequest.scala

@@ -58,7 +58,15 @@ case class PaymentRequest(prefix: String, amount: Option[MilliSatoshi], timestam
     case PaymentRequest.FallbackAddressTag(version, hash) if prefix == "lntb" => Bech32.encodeWitnessAddress("tb", version, hash)
   }
 
-  def routingInfo(): Seq[RoutingInfoTag] = tags.collect { case t: RoutingInfoTag => t}
+  def routingInfo(): Seq[RoutingInfoTag] = tags.collect { case t: RoutingInfoTag => t }
+
+  def expiry: Option[Long] = tags.collectFirst {
+    case PaymentRequest.ExpiryTag(seconds) => seconds
+  }
+
+  def minFinalCltvExpiry: Option[Long] = tags.collectFirst {
+    case PaymentRequest.MinFinalCltvExpiryTag(expiry) => expiry
+  }
 
   /**
     *
@@ -96,12 +104,16 @@ object PaymentRequest {
   // https://github.com/lightningnetwork/lightning-rfc/blob/master/02-peer-protocol.md#adding-an-htlc-update_add_htlc
   val maxAmount = MilliSatoshi(4294967296L)
 
-  def apply(chainHash: BinaryData, amount: Option[MilliSatoshi], paymentHash: BinaryData, privateKey: PrivateKey, description: String, fallbackAddress: Option[String] = None, expirySeconds: Option[Long] = None, timestamp: Long = System.currentTimeMillis() / 1000L): PaymentRequest = {
+  def apply(chainHash: BinaryData, amount: Option[MilliSatoshi], paymentHash: BinaryData, privateKey: PrivateKey,
+            description: String, fallbackAddress: Option[String] = None, expirySeconds: Option[Long] = None,
+            extraHops: Seq[Seq[ExtraHop]] = Nil, timestamp: Long = System.currentTimeMillis() / 1000L): PaymentRequest = {
+
     val prefix = chainHash match {
-    case Block.RegtestGenesisBlock.hash => "lntb"
-    case Block.TestnetGenesisBlock.hash => "lntb"
-    case Block.LivenetGenesisBlock.hash => "lnbc"
+      case Block.RegtestGenesisBlock.hash => "lntb"
+      case Block.TestnetGenesisBlock.hash => "lntb"
+      case Block.LivenetGenesisBlock.hash => "lnbc"
     }
+
     PaymentRequest(
       prefix = prefix,
       amount = amount,
@@ -110,7 +122,8 @@ object PaymentRequest {
       tags = List(
         Some(PaymentHashTag(paymentHash)),
         Some(DescriptionTag(description)),
-        expirySeconds.map(ExpiryTag(_))).flatten,
+        expirySeconds.map(ExpiryTag(_))
+      ).flatten ++ extraHops.map(RoutingInfoTag(_)),
       signature = BinaryData.empty)
       .sign(privateKey)
   }
@@ -200,29 +213,69 @@ object PaymentRequest {
   }
 
   /**
-    * Routing Info Tag
+    * Extra hop contained in RoutingInfoTag
     *
-    * @param pubkey          node id
-    * @param channelId       channel id
+    * @param nodeId          node id
+    * @param shortChannelId  channel id
     * @param fee             node fee
     * @param cltvExpiryDelta node cltv expiry delta
     */
-  case class RoutingInfoTag(pubkey: PublicKey, channelId: BinaryData, fee: Long, cltvExpiryDelta: Int) extends Tag {
+  case class ExtraHop(nodeId: PublicKey, shortChannelId: Long, fee: Long, cltvExpiryDelta: Int) extends PaymentHop {
+    def pack: Seq[Byte] = nodeId.toBin ++ Protocol.writeUInt64(shortChannelId, ByteOrder.BIG_ENDIAN) ++
+      Protocol.writeUInt64(fee, ByteOrder.BIG_ENDIAN) ++ Protocol.writeUInt16(cltvExpiryDelta, ByteOrder.BIG_ENDIAN)
+
+    // Fee is already pre-calculated for extra hops
+    def nextFee(msat: Long): Long = fee
+  }
+
+  /**
+    * Routing Info Tag
+    *
+    * @param path one or more entries containing extra routing information for a private route
+    */
+  case class RoutingInfoTag(path: Seq[ExtraHop]) extends Tag {
     override def toInt5s = {
-      val ints = Bech32.eight2five(pubkey.toBin ++ channelId ++ Protocol.writeUInt64(fee, ByteOrder.BIG_ENDIAN) ++ Protocol.writeUInt16(cltvExpiryDelta, ByteOrder.BIG_ENDIAN))
+      val ints = Bech32.eight2five(path.flatMap(_.pack))
       Seq(Bech32.map('r'), (ints.length / 32).toByte, (ints.length % 32).toByte) ++ ints
     }
   }
 
+  object RoutingInfoTag {
+    def parse(data: Seq[Byte]) = {
+      val pubkey = data.slice(0, 33)
+      val shortChannelId = Protocol.uint64(data.slice(33, 33 + 8), ByteOrder.BIG_ENDIAN)
+      val fee = Protocol.uint64(data.slice(33 + 8, 33 + 8 + 8), ByteOrder.BIG_ENDIAN)
+      val cltv = Protocol.uint16(data.slice(33 + 8 + 8, chunkLength), ByteOrder.BIG_ENDIAN)
+      ExtraHop(PublicKey(pubkey), shortChannelId, fee, cltv)
+    }
+
+    def parseAll(data: Seq[Byte]): Seq[ExtraHop] =
+      data.grouped(chunkLength).map(parse).toList
+
+    val chunkLength: Int = 33 + 8 + 8 + 2
+  }
+
   /**
     * Expiry Date
     *
-    * @param seconds expriry data for this payment request
+    * @param seconds expiry data for this payment request
     */
   case class ExpiryTag(seconds: Long) extends Tag {
     override def toInt5s = {
-      val ints = Seq((seconds / 32).toByte, (seconds % 32).toByte)
-      Seq(Bech32.map('x'), 0.toByte, 2.toByte) ++ ints
+      val ints = writeUnsignedLong(seconds)
+      Bech32.map('x') +: (writeSize(ints.size) ++ ints)
+    }
+  }
+
+  /**
+    * Min final CLTV expiry
+    *
+    * @param blocks min final cltv expiry, in blocks
+    */
+  case class MinFinalCltvExpiryTag(blocks: Long) extends Tag {
+    override def toInt5s = {
+      val ints = writeUnsignedLong(blocks)
+      Bech32.map('c') +: (writeSize(ints.size) ++ ints)
     }
   }
 
@@ -284,15 +337,14 @@ object PaymentRequest {
           }
         case r if r == Bech32.map('r') =>
           val data = Bech32.five2eight(input.drop(3).take(len))
-          val pubkey = PublicKey(data.take(33))
-          val channelId = data.drop(33).take(8)
-          val fee = Protocol.uint64(data.drop(33 + 8), ByteOrder.BIG_ENDIAN)
-          val cltv = Protocol.uint16(data.drop(33 + 8 + 8), ByteOrder.BIG_ENDIAN)
-          RoutingInfoTag(pubkey, channelId, fee, cltv)
+          val path = RoutingInfoTag.parseAll(data)
+          RoutingInfoTag(path)
         case x if x == Bech32.map('x') =>
-          require(len == 2, s"invalid length for expiry tag, should be 2 instead of $len")
-          val expiry = 32 * input(3) + input(4)
+          val expiry = readUnsignedLong(len, input.drop(3).take(len))
           ExpiryTag(expiry)
+        case c if c == Bech32.map('c') =>
+          val expiry = readUnsignedLong(len, input.drop(3).take(len))
+          MinFinalCltvExpiryTag(expiry)
       }
     }
   }
@@ -328,22 +380,24 @@ object PaymentRequest {
     }
   }
 
-  def toBits(value: Int5) : Seq[Bit] = Seq((value & 16) != 0, (value & 8) != 0, (value & 4) != 0, (value & 2) != 0, (value & 1) != 0)
+  def toBits(value: Int5): Seq[Bit] = Seq((value & 16) != 0, (value & 8) != 0, (value & 4) != 0, (value & 2) != 0, (value & 1) != 0)
 
   /**
     * write a 5bits integer to a stream
+    *
     * @param stream stream to write to
-    * @param value a 5bits value
+    * @param value  a 5bits value
     * @return an upated stream
     */
-  def write5(stream: BitStream, value: Int5) : BitStream = stream.writeBits(toBits(value))
+  def write5(stream: BitStream, value: Int5): BitStream = stream.writeBits(toBits(value))
 
   /**
     * read a 5bits value from a stream
+    *
     * @param stream stream to read from
     * @return a (stream, value) pair
     */
-  def read5(stream: BitStream) : (BitStream, Int5) = {
+  def read5(stream: BitStream): (BitStream, Int5) = {
     val (stream1, bits) = stream.readBits(5)
     val value = (if (bits(0)) 1 << 4 else 0) + (if (bits(1)) 1 << 3 else 0) + (if (bits(2)) 1 << 2 else 0) + (if (bits(3)) 1 << 1 else 0) + (if (bits(4)) 1 << 0 else 0)
     (stream1, (value & 0xff).toByte)
@@ -351,16 +405,58 @@ object PaymentRequest {
 
   /**
     * splits a bit stream into 5bits values
+    *
     * @param stream
     * @param acc
     * @return a sequence of 5bits values
     */
   @tailrec
-  def toInt5s(stream: BitStream, acc :Seq[Int5] = Nil) : Seq[Int5] = if (stream.bitCount == 0) acc else {
+  def toInt5s(stream: BitStream, acc: Seq[Int5] = Nil): Seq[Int5] = if (stream.bitCount == 0) acc else {
     val (stream1, value) = read5(stream)
     toInt5s(stream1, acc :+ value)
   }
 
+  /**
+    * prepend an unsigned long value to a sequence of Int5s
+    *
+    * @param value input value
+    * @param acc   sequence of Int5 values
+    * @return an update sequence of Int5s
+    */
+  @tailrec
+  def writeUnsignedLong(value: Long, acc: Seq[Int5] = Nil): Seq[Int5] = {
+    require(value >= 0)
+    if (value == 0) acc
+    else writeUnsignedLong(value / 32, (value % 32).toByte +: acc)
+  }
+
+  /**
+    * convert a tag data size to a sequence of Int5s. It * must * fit on a sequence
+    * of 2 Int5 values
+    *
+    * @param size data size
+    * @return size as a sequence of exactly 2 Int5 values
+    */
+  def writeSize(size: Long): Seq[Int5] = {
+    val output = writeUnsignedLong(size)
+    // make sure that size is encoded on 2 int5 values
+    output.length match {
+      case 0 => Seq(0.toByte, 0.toByte)
+      case 1 => 0.toByte +: output
+      case 2 => output
+      case n => throw new IllegalArgumentException("tag data length field must be encoded on 2 5-bits integers")
+    }
+  }
+
+  /**
+    * reads an unsigned long value from a sequence of Int5s
+    *
+    * @param length length of the sequence
+    * @param ints   sequence of Int5s
+    * @return an unsigned long value
+    */
+  def readUnsignedLong(length: Int, ints: Seq[Int5]): Long = ints.take(length).foldLeft(0L) { case (acc, i) => acc * 32 + i }
+
   /**
     *
     * @param input bech32-encoded payment request
@@ -377,10 +473,10 @@ object PaymentRequest {
     val data1 = data0.drop(7)
 
     @tailrec
-    def loop(data: Seq[Int5], tags: Seq[Seq[Int5]] = Nil): Seq[Seq[Int5]] = if(data.isEmpty) tags else {
+    def loop(data: Seq[Int5], tags: Seq[Seq[Int5]] = Nil): Seq[Seq[Int5]] = if (data.isEmpty) tags else {
       // 104 is the size of a signature
-        val len = 1 + 2 + 32 * data(1) + data(2)
-        loop(data.drop(len), tags :+ data.take(len))
+      val len = 1 + 2 + 32 * data(1) + data(2)
+      loop(data.drop(len), tags :+ data.take(len))
     }
 
     val rawtags = loop(data1)

eclair-core/src/main/scala/fr/acinq/eclair/payment/Relayer.scala

@@ -1,14 +1,12 @@
 package fr.acinq.eclair.payment
 
-import akka.actor.{Actor, ActorLogging, ActorRef, Props}
-import fr.acinq.bitcoin.Crypto.{PrivateKey, ripemd160, sha256}
-import fr.acinq.bitcoin.{BinaryData, Crypto, MilliSatoshi, ScriptWitness, Transaction}
-import fr.acinq.eclair.Globals
-import fr.acinq.eclair.blockchain.WatchEventSpent
+import akka.actor.{Actor, ActorLogging, ActorRef, Props, Status}
+import fr.acinq.bitcoin.{BinaryData, Crypto, MilliSatoshi}
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.Sphinx
 import fr.acinq.eclair.router.Announcements
 import fr.acinq.eclair.wire._
+import fr.acinq.eclair.{Globals, NodeParams}
 import scodec.bits.BitVector
 import scodec.{Attempt, DecodeResult}
 
@@ -17,16 +15,16 @@ import scala.util.{Failure, Success, Try}
 // @formatter:off
 
 sealed trait Origin
-case class Local(sender: ActorRef) extends Origin
-case class Relayed(upstream: ActorRef, htlcIn: UpdateAddHtlc) extends Origin
+case class Local(sender: Option[ActorRef]) extends Origin // we don't persist reference to local actors
+case class Relayed(originChannelId: BinaryData, originHtlcId: Long, amountMsatIn: Long, amountMsatOut: Long) extends Origin
 
-case class AddHtlcSucceeded(add: UpdateAddHtlc, origin: Origin)
-case class AddHtlcFailed(add: CMD_ADD_HTLC, error: ChannelException)
-case class AddHtlcDiscarded(add: UpdateAddHtlc) // dropped because of disconnection
 case class ForwardAdd(add: UpdateAddHtlc)
-case class ForwardFulfill(fulfill: UpdateFulfillHtlc)
-case class ForwardFail(fail: UpdateFailHtlc)
-case class ForwardFailMalformed(fail: UpdateFailMalformedHtlc)
+case class ForwardFulfill(fulfill: UpdateFulfillHtlc, to: Origin)
+case class ForwardLocalFail(error: Throwable, to: Origin) // happens when the failure happened in a local channel (and not in some downstream channel)
+case class ForwardFail(fail: UpdateFailHtlc, to: Origin)
+case class ForwardFailMalformed(fail: UpdateFailMalformedHtlc, to: Origin)
+
+case class AckFulfillCmd(channelId: BinaryData, htlcId: Long)
 
 // @formatter:on
 
@@ -34,43 +32,33 @@ case class ForwardFailMalformed(fail: UpdateFailMalformedHtlc)
 /**
   * Created by PM on 01/02/2017.
   */
-class Relayer(nodeSecret: PrivateKey, paymentHandler: ActorRef) extends Actor with ActorLogging {
+class Relayer(nodeParams: NodeParams, register: ActorRef, paymentHandler: ActorRef) extends Actor with ActorLogging {
+
+  import nodeParams.preimagesDb
 
   context.system.eventStream.subscribe(self, classOf[ChannelStateChanged])
-  context.system.eventStream.subscribe(self, classOf[ShortChannelIdAssigned])
 
-  override def receive: Receive = main(Map(), Map(), Map(), Map())
+  override def receive: Receive = main(Map())
 
-  def shortId2Channel(channels: Map[BinaryData, ActorRef], shortIds: Map[Long, BinaryData], shortId: Long): Option[ActorRef] = shortIds.get(shortId).flatMap(channels.get(_))
+  def main(channelUpdates: Map[Long, ChannelUpdate]): Receive = {
 
-  def main(channels: Map[BinaryData, ActorRef], shortIds: Map[Long, BinaryData], bindings: Map[UpdateAddHtlc, Origin], channelUpdates: Map[Long, ChannelUpdate]): Receive = {
-
-    case ChannelStateChanged(channel, _, _, _, NORMAL, d: DATA_NORMAL) =>
-      import d.commitments.channelId
-      log.info(s"adding channel $channelId to available channels")
-      context become main(channels + (channelId -> channel), shortIds, bindings, channelUpdates)
-
-    case ChannelStateChanged(_, _, _, _, NEGOTIATING, d: DATA_NEGOTIATING) =>
-      import d.commitments.channelId
-      log.info(s"removing channel $channelId from available channels")
-      // TODO: cleanup bindings
-      context become main(channels - channelId, shortIds, bindings, channelUpdates)
-
-    case ChannelStateChanged(_, _, _, _, CLOSING, d: DATA_CLOSING) =>
-      import d.commitments.channelId
-      log.info(s"removing channel $channelId from available channels")
-      // TODO: cleanup bindings
-      context become main(channels - channelId, shortIds, bindings, channelUpdates)
-
-    case ShortChannelIdAssigned(_, channelId, shortChannelId) =>
-      context become main(channels, shortIds + (shortChannelId -> channelId), bindings, channelUpdates)
+    case ChannelStateChanged(channel, _, _, _, NORMAL | SHUTDOWN | CLOSING, d: HasCommitments) =>
+      import d.channelId
+      preimagesDb.listPreimages(channelId) match {
+        case Nil => {}
+        case preimages =>
+          log.info(s"re-sending ${preimages.size} unacked fulfills to channel $channelId")
+          preimages.map(p => CMD_FULFILL_HTLC(p._2, p._3, commit = false)).foreach(channel ! _)
+          // better to sign once instead of after each fulfill
+          channel ! CMD_SIGN
+      }
 
     case channelUpdate: ChannelUpdate =>
       log.info(s"updating relay parameters with channelUpdate=$channelUpdate")
-      context become main(channels, shortIds, bindings, channelUpdates + (channelUpdate.shortChannelId -> channelUpdate))
+      context become main(channelUpdates + (channelUpdate.shortChannelId -> channelUpdate))
 
     case ForwardAdd(add) =>
-      Try(Sphinx.parsePacket(nodeSecret, add.paymentHash, add.onionRoutingPacket))
+      Try(Sphinx.parsePacket(nodeParams.privateKey, add.paymentHash, add.onionRoutingPacket))
         .map {
           case Sphinx.ParsedPacket(payload, nextPacket, sharedSecret) => (LightningMessageCodecs.perHopPayloadCodec.decode(BitVector(payload.data)), nextPacket, sharedSecret)
         } match {
@@ -81,34 +69,28 @@ class Relayer(nodeSecret: PrivateKey, paymentHandler: ActorRef) extends Actor wi
               sender ! CMD_FAIL_HTLC(add.id, Right(FinalIncorrectHtlcAmount(add.amountMsat)), commit = true)
             case PerHopPayload(_, _, finalOutgoingCltvValue) if finalOutgoingCltvValue != add.expiry =>
               sender ! CMD_FAIL_HTLC(add.id, Right(FinalIncorrectCltvExpiry(add.expiry)), commit = true)
-            case _  if add.expiry < Globals.blockCount.get() + 3 => // TODO: check hardcoded value
+            case _ if add.expiry < Globals.blockCount.get() + 3 => // TODO: check hardcoded value
               sender ! CMD_FAIL_HTLC(add.id, Right(FinalExpiryTooSoon), commit = true)
             case _ =>
               paymentHandler forward add
           }
         case Success((Attempt.Successful(DecodeResult(perHopPayload, _)), nextPacket, _)) =>
-          shortId2Channel(channels, shortIds, perHopPayload.channel_id) match {
-            case Some(downstream) =>
-              val channelUpdate_opt = channelUpdates.get(perHopPayload.channel_id)
-              channelUpdate_opt match {
-                case None =>
-                  // TODO: clarify what we're supposed to do in the specs
-                  sender ! CMD_FAIL_HTLC(add.id, Right(TemporaryNodeFailure), commit = true)
-                case Some(channelUpdate) if !Announcements.isEnabled(channelUpdate.flags) =>
-                  sender ! CMD_FAIL_HTLC(add.id, Right(ChannelDisabled(channelUpdate.flags, channelUpdate)), commit = true)
-                case Some(channelUpdate) if add.amountMsat < channelUpdate.htlcMinimumMsat =>
-                  sender ! CMD_FAIL_HTLC(add.id, Right(AmountBelowMinimum(add.amountMsat, channelUpdate)), commit = true)
-                case Some(channelUpdate) if add.expiry != perHopPayload.outgoingCltvValue + channelUpdate.cltvExpiryDelta =>
-                  sender ! CMD_FAIL_HTLC(add.id, Right(IncorrectCltvExpiry(add.expiry, channelUpdate)), commit = true)
-                case Some(channelUpdate) if add.expiry < Globals.blockCount.get() + 3 => // TODO: hardcoded value
-                  sender ! CMD_FAIL_HTLC(add.id, Right(ExpiryTooSoon(channelUpdate)), commit = true)
-                case _ =>
-                  log.info(s"forwarding htlc #${add.id} to downstream=$downstream")
-                  downstream forward CMD_ADD_HTLC(perHopPayload.amtToForward, add.paymentHash, perHopPayload.outgoingCltvValue, nextPacket.serialize, upstream_opt = Some(add), commit = true)
-              }
+          val channelUpdate_opt = channelUpdates.get(perHopPayload.channel_id)
+          channelUpdate_opt match {
             case None =>
-              log.warning(s"couldn't resolve downstream channel ${perHopPayload.channel_id}, failing htlc #${add.id}")
-              sender ! CMD_FAIL_HTLC(add.id, Right(UnknownNextPeer), commit = true)
+              // TODO: clarify what we're supposed to do in the specs
+              sender ! CMD_FAIL_HTLC(add.id, Right(TemporaryNodeFailure), commit = true)
+            case Some(channelUpdate) if !Announcements.isEnabled(channelUpdate.flags) =>
+              sender ! CMD_FAIL_HTLC(add.id, Right(ChannelDisabled(channelUpdate.flags, channelUpdate)), commit = true)
+            case Some(channelUpdate) if add.amountMsat < channelUpdate.htlcMinimumMsat =>
+              sender ! CMD_FAIL_HTLC(add.id, Right(AmountBelowMinimum(add.amountMsat, channelUpdate)), commit = true)
+            case Some(channelUpdate) if add.expiry != perHopPayload.outgoingCltvValue + channelUpdate.cltvExpiryDelta =>
+              sender ! CMD_FAIL_HTLC(add.id, Right(IncorrectCltvExpiry(add.expiry, channelUpdate)), commit = true)
+            case Some(channelUpdate) if add.expiry < Globals.blockCount.get() + 3 => // TODO: hardcoded value
+              sender ! CMD_FAIL_HTLC(add.id, Right(ExpiryTooSoon(channelUpdate)), commit = true)
+            case _ =>
+              log.info(s"forwarding htlc #${add.id} to shortChannelId=${perHopPayload.channel_id}")
+              register forward Register.ForwardShortId(perHopPayload.channel_id, CMD_ADD_HTLC(perHopPayload.amtToForward, add.paymentHash, perHopPayload.outgoingCltvValue, nextPacket.serialize, upstream_opt = Some(add), commit = true))
           }
         case Success((Attempt.Failure(cause), _, _)) =>
           log.error(s"couldn't parse payload: $cause")
@@ -119,124 +101,53 @@ class Relayer(nodeSecret: PrivateKey, paymentHandler: ActorRef) extends Actor wi
           sender ! CMD_FAIL_MALFORMED_HTLC(add.id, Crypto.sha256(add.onionRoutingPacket), failureCode = FailureMessageCodecs.BADONION, commit = true)
       }
 
-    case AddHtlcSucceeded(downstream, origin) =>
-      origin match {
-        case Local(_) => log.info(s"we are the origin of htlc ${downstream.channelId}/${downstream.id}")
-        case Relayed(_, upstream) => log.info(s"relayed htlc ${upstream.channelId}/${upstream.id} to ${downstream.channelId}/${downstream.id}")
-      }
-      context become main(channels, shortIds, bindings + (downstream -> origin), channelUpdates)
+    case Register.ForwardShortIdFailure(Register.ForwardShortId(shortChannelId, CMD_ADD_HTLC(_, _, _, _, Some(add), _))) =>
+      log.warning(s"couldn't resolve downstream channel $shortChannelId, failing htlc #${add.id}")
+      register ! Register.Forward(add.channelId, CMD_FAIL_HTLC(add.id, Right(UnknownNextPeer), commit = true))
 
-    case AddHtlcFailed(CMD_ADD_HTLC(_, _, _, _, Some(updateAddHtlc), _), error) if channels.contains(updateAddHtlc.channelId) =>
-      val upstream = channels(updateAddHtlc.channelId)
-      val channelUpdate_opt = for {
-        channelId <- channels.map(_.swap).get(sender)
-        shortId <- shortIds.map(_.swap).get(channelId)
-        update <- channelUpdates.get(shortId)
-      } yield update
-      // detail errors are caught before relaying the htlc to the downstream channel, here we just return generic error messages
-      channelUpdate_opt match {
-        case None =>
-          // TODO: clarify what we're supposed to do in the specs
-          upstream ! CMD_FAIL_HTLC(updateAddHtlc.id, Right(TemporaryNodeFailure), commit = true)
-        case Some(channelUpdate) =>
-          upstream ! CMD_FAIL_HTLC(updateAddHtlc.id, Right(TemporaryChannelFailure(channelUpdate)), commit = true)
-      }
+    case ForwardFulfill(fulfill, Local(Some(sender))) =>
+      sender ! fulfill
 
-    case AddHtlcDiscarded(add) =>
-      bindings.find(b => b._1.channelId == add.channelId && b._1.id == add.id) match {
-        case Some((htlcOut, Relayed(upstream, htlcIn))) =>
-          // TODO: fail htlc upstream
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case Some((htlcOut, Local(origin))) =>
-          log.info(s"we were the origin payer for htlc #${add.id}")
-          origin ! 'cancelled
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case None =>
-          log.warning(s"no origin found for htlc ${add.channelId}/${add.id}")
-      }
+    case ForwardFulfill(fulfill, Relayed(originChannelId, originHtlcId, amountMsatIn, amountMsatOut)) =>
+      val cmd = CMD_FULFILL_HTLC(originHtlcId, fulfill.paymentPreimage, commit = true)
+      register ! Register.Forward(originChannelId, cmd)
+      context.system.eventStream.publish(PaymentRelayed(MilliSatoshi(amountMsatIn), MilliSatoshi(amountMsatOut), Crypto.sha256(fulfill.paymentPreimage)))
+      // we also store the preimage in a db (note that this happens *after* forwarding the fulfill to the channel, so we don't add latency)
+      preimagesDb.addPreimage(originChannelId, originHtlcId, fulfill.paymentPreimage)
 
-    case ForwardFulfill(fulfill) =>
-      bindings.find(b => b._1.channelId == fulfill.channelId && b._1.id == fulfill.id) match {
-        case Some((htlcOut, Relayed(upstream, htlcIn))) =>
-          upstream ! CMD_FULFILL_HTLC(htlcIn.id, fulfill.paymentPreimage, commit = true)
-          context.system.eventStream.publish(PaymentRelayed(MilliSatoshi(htlcIn.amountMsat), MilliSatoshi(htlcIn.amountMsat - htlcOut.amountMsat), htlcIn.paymentHash))
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case Some((htlcOut, Local(origin))) =>
-          log.info(s"we were the origin payer for htlc #${fulfill.id}")
-          origin ! fulfill
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case None =>
-          log.warning(s"no origin found for htlc ${fulfill.channelId}/${fulfill.id}")
-      }
+    case AckFulfillCmd(channelId, htlcId) =>
+      log.debug(s"fulfill acked for channelId=$channelId htlcId=$htlcId")
+      preimagesDb.removePreimage(channelId, htlcId)
 
-    case ForwardFail(fail) =>
-      bindings.find(b => b._1.channelId == fail.channelId && b._1.id == fail.id) match {
-        case Some((htlcOut, Relayed(upstream, htlcIn))) =>
-          upstream ! CMD_FAIL_HTLC(htlcIn.id, Left(fail.reason), commit = true)
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case Some((htlcOut, Local(origin))) =>
-          log.info(s"we were the origin payer for htlc #${fail.id}")
-          origin ! fail
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case None =>
-          log.warning(s"no origin found for htlc ${fail.channelId}/${fail.id}")
-      }
+    case ForwardLocalFail(error, Local(Some(sender))) =>
+      sender ! Status.Failure(error)
 
-    case ForwardFailMalformed(fail) =>
-      bindings.find(b => b._1.channelId == fail.channelId && b._1.id == fail.id) match {
-        case Some((htlcOut, Relayed(upstream, htlcIn))) =>
-          upstream ! CMD_FAIL_MALFORMED_HTLC(htlcIn.id, fail.onionHash, fail.failureCode, commit = true)
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case Some((htlcOut, Local(origin))) =>
-          log.info(s"we were the origin payer for htlc #${fail.id}")
-          origin ! fail
-          context become main(channels, shortIds, bindings - htlcOut, channelUpdates)
-        case None =>
-          log.warning(s"no origin found for htlc ${fail.channelId}/${fail.id}")
+    case ForwardLocalFail(error, Relayed(originChannelId, originHtlcId, _, _)) =>
+      // TODO: clarify what we're supposed to do in the specs depending on the error
+      val failure = error match {
+        case HtlcTimedout(_) => PermanentChannelFailure
+        case _ => TemporaryNodeFailure
       }
+      val cmd = CMD_FAIL_HTLC(originHtlcId, Right(failure), commit = true)
+      register ! Register.Forward(originChannelId, cmd)
 
-    case w@WatchEventSpent(BITCOIN_HTLC_SPENT, tx) =>
-      // when a remote or local commitment tx containing outgoing htlcs is published on the network,
-      // we watch it in order to extract payment preimage if funds are pulled by the counterparty
-      // we can then use these preimages to fulfill origin htlcs
-      log.warning(s"processing BITCOIN_HTLC_SPENT with txid=${tx.txid} tx=${Transaction.write(tx)}")
-      require(tx.txIn.size == 1, s"htlc tx should only have 1 input")
-      val witness = tx.txIn(0).witness
-      val extracted = witness match {
-        case ScriptWitness(Seq(localSig, paymentPreimage, htlcOfferedScript)) if paymentPreimage.size == 32 =>
-          log.warning(s"extracted preimage=$paymentPreimage from tx=${Transaction.write(tx)} (claim-htlc-success)")
-          paymentPreimage
-        case ScriptWitness(Seq(BinaryData.empty, remoteSig, localSig, paymentPreimage, htlcReceivedScript)) if paymentPreimage.size == 32 =>
-          log.warning(s"extracted preimage=$paymentPreimage from tx=${Transaction.write(tx)} (htlc-success)")
-          paymentPreimage
-        case ScriptWitness(Seq(BinaryData.empty, remoteSig, localSig, BinaryData.empty, htlcOfferedScript)) =>
-          val paymentHash160 = BinaryData(htlcOfferedScript.slice(109, 109 + 20))
-          log.warning(s"extracted paymentHash160=$paymentHash160 from tx=${Transaction.write(tx)} (htlc-timeout)")
-          paymentHash160
-        case ScriptWitness(Seq(remoteSig, BinaryData.empty, htlcReceivedScript)) =>
-          val paymentHash160 = BinaryData(htlcReceivedScript.slice(69, 69 + 20))
-          log.warning(s"extracted paymentHash160=$paymentHash160 from tx=${Transaction.write(tx)} (claim-htlc-timeout)")
-          paymentHash160
-      }
-      // TODO: should we handle local htlcs here as well? currently timed out htlcs that we sent will never have an answer
-      // TODO: we do not handle the case where htlcs transactions end up beeing unconfirmed this can happen if an htlc-success
-      // tx is published right before a htlc timed out
-      val htlcsOut = bindings.collect {
-        case b@(htlcOut, Relayed(upstream, htlcIn)) if htlcIn.paymentHash == sha256(extracted) =>
-          log.warning(s"found a match between preimage=$extracted and origin htlc=$htlcIn")
-          upstream ! CMD_FULFILL_HTLC(htlcIn.id, extracted, commit = true)
-          htlcOut
-        case b@(htlcOut, Relayed(upstream, htlcIn)) if ripemd160(htlcIn.paymentHash) == extracted =>
-          log.warning(s"found a match between paymentHash160=$extracted and origin htlc=$htlcIn")
-          upstream ! CMD_FAIL_HTLC(htlcIn.id, Right(PermanentChannelFailure), commit = true)
-          htlcOut
-      }
-      context become main(channels, shortIds, bindings -- htlcsOut, channelUpdates)
+    case ForwardFail(fail, Local(Some(sender))) =>
+      sender ! fail
 
-    case 'channels => sender ! channels
+    case ForwardFail(fail, Relayed(originChannelId, originHtlcId, _, _)) =>
+      val cmd = CMD_FAIL_HTLC(originHtlcId, Left(fail.reason), commit = true)
+      register ! Register.Forward(originChannelId, cmd)
+
+    case ForwardFailMalformed(fail, Local(Some(sender))) =>
+      sender ! fail
+
+    case ForwardFailMalformed(fail, Relayed(originChannelId, originHtlcId, _, _)) =>
+      val cmd = CMD_FAIL_MALFORMED_HTLC(originHtlcId, fail.onionHash, fail.failureCode, commit = true)
+      register ! Register.Forward(originChannelId, cmd)
   }
+
 }
 
 object Relayer {
-  def props(nodeSecret: PrivateKey, paymentHandler: ActorRef) = Props(classOf[Relayer], nodeSecret: PrivateKey, paymentHandler)
+  def props(nodeParams: NodeParams, register: ActorRef, paymentHandler: ActorRef) = Props(classOf[Relayer], nodeParams, register, paymentHandler)
 }

eclair-core/src/main/scala/fr/acinq/eclair/router/Announcements.scala

@@ -79,6 +79,7 @@ object Announcements {
     * The creating node MUST set node-id-1 and node-id-2 to the public keys of the
     * two nodes who are operating the channel, such that node-id-1 is the numerically-lesser
     * of the two DER encoded keys sorted in ascending numerical order,
+    *
     * @return true if localNodeId is node1
     */
   def isNode1(localNodeId: BinaryData, remoteNodeId: BinaryData) = LexicographicalOrdering.isLessThan(localNodeId, remoteNodeId)
@@ -87,6 +88,7 @@ object Announcements {
     * BOLT 7:
     * The creating node [...] MUST set the direction bit of flags to 0 if
     * the creating node is node-id-1 in that message, otherwise 1.
+    *
     * @return true if the node who sent these flags is node1
     */
   def isNode1(flags: BinaryData) = !BitVector(flags.data).reverse.get(0)
@@ -94,6 +96,7 @@ object Announcements {
   /**
     * A node MAY create and send a channel_update with the disable bit set to
     * signal the temporary unavailability of a channel
+    *
     * @return
     */
   def isEnabled(flags: BinaryData) = !BitVector(flags.data).reverse.get(1)

eclair-core/src/main/scala/fr/acinq/eclair/router/NetworkEvents.scala

@@ -1,6 +1,7 @@
 package fr.acinq.eclair.router
 
-import fr.acinq.bitcoin.{BinaryData, Satoshi}
+import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.bitcoin.Satoshi
 import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement}
 
 /**
@@ -12,7 +13,7 @@ case class NodeDiscovered(ann: NodeAnnouncement) extends NetworkEvent
 
 case class NodeUpdated(ann: NodeAnnouncement) extends NetworkEvent
 
-case class NodeLost(nodeId: BinaryData) extends NetworkEvent
+case class NodeLost(nodeId: PublicKey) extends NetworkEvent
 
 case class ChannelDiscovered(ann: ChannelAnnouncement, capacity: Satoshi) extends NetworkEvent
 

eclair-core/src/main/scala/fr/acinq/eclair/router/Router.scala

@@ -11,6 +11,7 @@ import fr.acinq.eclair._
 import fr.acinq.eclair.blockchain._
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.io.Peer
+import fr.acinq.eclair.payment.Hop
 import fr.acinq.eclair.transactions.Scripts
 import fr.acinq.eclair.wire._
 import org.jgrapht.alg.shortestpath.DijkstraShortestPath
@@ -19,13 +20,13 @@ import org.jgrapht.graph.{DefaultDirectedGraph, DefaultEdge, SimpleGraph}
 
 import scala.collection.JavaConversions._
 import scala.compat.Platform
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Random, Success, Try}
 
 // @formatter:off
 
 case class ChannelDesc(id: Long, a: PublicKey, b: PublicKey)
-case class Hop(nodeId: PublicKey, nextNodeId: PublicKey, lastUpdate: ChannelUpdate)
 case class RouteRequest(source: PublicKey, target: PublicKey, ignoreNodes: Set[PublicKey] = Set.empty, ignoreChannels: Set[Long] = Set.empty)
 case class RouteResponse(hops: Seq[Hop], ignoreNodes: Set[PublicKey], ignoreChannels: Set[Long]) { require(hops.size > 0, "route cannot be empty") }
 case class ExcludeChannel(desc: ChannelDesc) // this is used when we get a TemporaryChannelFailure, to give time for the channel to recover (note that exclusions are directed)
@@ -47,6 +48,10 @@ sealed trait State
 case object NORMAL extends State
 case object WAITING_FOR_VALIDATION extends State
 
+case object TickBroadcast
+case object TickValidate
+case object TickPruneStaleChannels
+
 // @formatter:on
 
 /**
@@ -59,8 +64,18 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
 
   import ExecutionContext.Implicits.global
 
+  context.system.eventStream.subscribe(self, classOf[ChannelStateChanged])
+
+  setTimer(TickBroadcast.toString, TickBroadcast, nodeParams.routerBroadcastInterval, repeat = true)
+  setTimer(TickValidate.toString, TickValidate, nodeParams.routerValidateInterval, repeat = true)
+  setTimer(TickPruneStaleChannels.toString, TickPruneStaleChannels, 1 day, repeat = true)
+
   val db = nodeParams.networkDb
 
+  // Note: We go through the whole validation process instead of directly loading into memory, because the channels
+  // could have been closed while we were shutdown, and if someone connects to us right after startup we don't want to
+  // advertise invalid channels. We could optimize this (at least not fetch txes from the blockchain, and not check sigs)
+  log.info(s"loading network announcements from db...")
   db.listChannels().map(self ! _)
   db.listNodes().map(self ! _)
   db.listChannelUpdates().map(self ! _)
@@ -68,17 +83,12 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
     val nodeAnn = Announcements.makeNodeAnnouncement(nodeParams.privateKey, nodeParams.alias, nodeParams.color, nodeParams.publicAddresses, Platform.currentTime / 1000)
     self ! nodeAnn
   }
-
-  context.system.eventStream.subscribe(self, classOf[ChannelStateChanged])
-
-  setTimer("broadcast", 'tick_broadcast, nodeParams.routerBroadcastInterval, repeat = true)
-  setTimer("validate", 'tick_validate, nodeParams.routerValidateInterval, repeat = true)
+  log.info(s"starting state machine")
 
   startWith(NORMAL, Data(Map.empty, Map.empty, Map.empty, Nil, Nil, Nil, Map.empty, Map.empty, Set.empty))
 
   when(NORMAL) {
-
-    case Event('tick_validate, d) =>
+    case Event(TickValidate, d) =>
       require(d.awaiting.size == 0)
       var i = 0
       // we extract a batch of channel announcements from the stash
@@ -96,7 +106,6 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
   }
 
   when(WAITING_FOR_VALIDATION) {
-
     case Event(ParallelGetResponse(results), d) =>
       val validated = results.map {
         case IndividualResult(c, Some(tx), true) =>
@@ -111,7 +120,7 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
             log.error(s"invalid script for shortChannelId=${c.shortChannelId} txid=${tx.txid} ann=$c")
             None
           } else {
-            watcher ! WatchSpentBasic(self, tx.txid, outputIndex, BITCOIN_FUNDING_OTHER_CHANNEL_SPENT(c.shortChannelId))
+            watcher ! WatchSpentBasic(self, tx, outputIndex, BITCOIN_FUNDING_EXTERNAL_CHANNEL_SPENT(c.shortChannelId))
             // TODO: check feature bit set
             log.debug(s"added channel channelId=${c.shortChannelId}")
             context.system.eventStream.publish(ChannelDiscovered(c, tx.txOut(outputIndex).amount))
@@ -131,7 +140,7 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
       }.flatten
       // we reprocess node and channel-update announcements that may have been validated
       val (resend, stash1) = d.stash.partition {
-        case n: NodeAnnouncement => results.exists(r => isRelatedTo(r.c, n))
+        case n: NodeAnnouncement => results.exists(r => isRelatedTo(r.c, n.nodeId))
         case u: ChannelUpdate => results.exists(r => r.c.shortChannelId == u.shortChannelId)
         case _ => false
       }
@@ -140,55 +149,53 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
   }
 
   whenUnhandled {
-
     case Event(ChannelStateChanged(_, _, _, _, channel.NORMAL, d: DATA_NORMAL), d1) =>
       stay using d1.copy(localChannels = d1.localChannels + (d.commitments.channelId -> d.commitments.remoteParams.nodeId))
 
     case Event(ChannelStateChanged(_, _, _, channel.NORMAL, _, d: DATA_NEGOTIATING), d1) =>
       stay using d1.copy(localChannels = d1.localChannels - d.commitments.channelId)
 
-    case Event(c: ChannelStateChanged, _) => stay
+    case Event(_: ChannelStateChanged, _) => stay
 
     case Event(SendRoutingState(remote), Data(nodes, channels, updates, _, _, _, _, _, _)) =>
       log.debug(s"info sending all announcements to $remote: channels=${channels.size} nodes=${nodes.size} updates=${updates.size}")
-      channels.values.foreach(remote ! _)
-      nodes.values.foreach(remote ! _)
-      updates.values.foreach(remote ! _)
+      // we group and add delays to leave room for channel messages
+      context.actorOf(ThrottleForwarder.props(remote, channels.values ++ nodes.values ++ updates.values, 100, 100 millis))
       stay
 
     case Event(c: ChannelAnnouncement, d) =>
       log.debug(s"received channel announcement for shortChannelId=${c.shortChannelId} nodeId1=${c.nodeId1} nodeId2=${c.nodeId2}")
-      if (!Announcements.checkSigs(c)) {
+      if (d.channels.containsKey(c.shortChannelId) || d.awaiting.exists(_.shortChannelId == c.shortChannelId) || d.stash.contains(c)) {
+        log.debug(s"ignoring $c (duplicate)")
+        stay
+      } else if (!Announcements.checkSigs(c)) {
         log.error(s"bad signature for announcement $c")
         sender ! Error(Peer.CHANNELID_ZERO, "bad announcement sig!!!".getBytes())
         stay
-      } else if (d.channels.containsKey(c.shortChannelId) || d.awaiting.exists(_.shortChannelId == c.shortChannelId) || d.stash.contains(c)) {
-        log.debug(s"ignoring $c (duplicate)")
-        stay
       } else {
         log.debug(s"stashing $c")
         stay using d.copy(stash = d.stash :+ c, origins = d.origins + (c -> sender))
       }
 
     case Event(n: NodeAnnouncement, d: Data) =>
-      if (!Announcements.checkSig(n)) {
+      if (d.nodes.containsKey(n.nodeId) && d.nodes(n.nodeId).timestamp >= n.timestamp) {
+        log.debug(s"ignoring announcement $n (old timestamp or duplicate)")
+        stay
+      } else if (!Announcements.checkSig(n)) {
         log.error(s"bad signature for announcement $n")
         sender ! Error(Peer.CHANNELID_ZERO, "bad announcement sig!!!".getBytes())
         stay
-      } else if (d.nodes.containsKey(n.nodeId) && d.nodes(n.nodeId).timestamp >= n.timestamp) {
-        log.debug(s"ignoring announcement $n (old timestamp or duplicate)")
-        stay
       } else if (d.nodes.containsKey(n.nodeId)) {
         log.debug(s"updated node nodeId=${n.nodeId}")
         context.system.eventStream.publish(NodeUpdated(n))
         db.updateNode(n)
         stay using d.copy(nodes = d.nodes + (n.nodeId -> n), rebroadcast = d.rebroadcast :+ n, origins = d.origins + (n -> sender))
-      } else if (d.channels.values.exists(c => isRelatedTo(c, n))) {
+      } else if (d.channels.values.exists(c => isRelatedTo(c, n.nodeId))) {
         log.debug(s"added node nodeId=${n.nodeId}")
         context.system.eventStream.publish(NodeDiscovered(n))
         db.addNode(n)
         stay using d.copy(nodes = d.nodes + (n.nodeId -> n), rebroadcast = d.rebroadcast :+ n, origins = d.origins + (n -> sender))
-      } else if (d.awaiting.exists(c => isRelatedTo(c, n)) || d.stash.collectFirst { case c: ChannelAnnouncement if isRelatedTo(c, n) => c }.isDefined) {
+      } else if (d.awaiting.exists(c => isRelatedTo(c, n.nodeId)) || d.stash.collectFirst { case c: ChannelAnnouncement if isRelatedTo(c, n.nodeId) => c }.isDefined) {
         log.debug(s"stashing $n")
         stay using d.copy(stash = d.stash :+ n, origins = d.origins + (n -> sender))
       } else {
@@ -202,14 +209,14 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
       if (d.channels.contains(u.shortChannelId)) {
         val c = d.channels(u.shortChannelId)
         val desc = getDesc(u, c)
-        if (!Announcements.checkSig(u, getDesc(u, d.channels(u.shortChannelId)).a)) {
+        if (d.updates.contains(desc) && d.updates(desc).timestamp >= u.timestamp) {
+          log.debug(s"ignoring $u (old timestamp or duplicate)")
+          stay
+        } else if (!Announcements.checkSig(u, getDesc(u, d.channels(u.shortChannelId)).a)) {
           // TODO: (dirty) this will make the origin channel close the connection
           log.error(s"bad signature for announcement $u")
           sender ! Error(Peer.CHANNELID_ZERO, "bad announcement sig!!!".getBytes())
           stay
-        } else if (d.updates.contains(desc) && d.updates(desc).timestamp >= u.timestamp) {
-          log.debug(s"ignoring $u (old timestamp or duplicate)")
-          stay
         } else if (d.updates.contains(desc)) {
           log.debug(s"updated $u")
           context.system.eventStream.publish(ChannelUpdateReceived(u))
@@ -229,30 +236,28 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
         stay
       }
 
-    case Event(WatchEventSpentBasic(BITCOIN_FUNDING_OTHER_CHANNEL_SPENT(shortChannelId)), d)
+    case Event(WatchEventSpentBasic(BITCOIN_FUNDING_EXTERNAL_CHANNEL_SPENT(shortChannelId)), d)
       if d.channels.containsKey(shortChannelId) =>
       val lostChannel = d.channels(shortChannelId)
-      log.debug(s"funding tx of channelId=$shortChannelId has been spent")
-      log.debug(s"removed channel channelId=$shortChannelId")
-      context.system.eventStream.publish(ChannelLost(shortChannelId))
-
-      def isNodeLost(nodeId: PublicKey): Option[PublicKey] = {
-        // has nodeId still open channels?
-        if ((d.channels - shortChannelId).values.filter(c => c.nodeId1 == nodeId || c.nodeId2 == nodeId).isEmpty) {
-          context.system.eventStream.publish(NodeLost(nodeId))
-          log.debug(s"removed node nodeId=$nodeId")
-          Some(nodeId)
-        } else None
-      }
-
-      val lostNodes = isNodeLost(lostChannel.nodeId1).toSeq ++ isNodeLost(lostChannel.nodeId2).toSeq
+      log.info(s"funding tx of channelId=$shortChannelId has been spent")
+      // we need to remove nodes that aren't tied to any channels anymore
+      val channels1 = d.channels - lostChannel.shortChannelId
+      val lostNodes = Seq(lostChannel.nodeId1, lostChannel.nodeId2).filterNot(nodeId => hasChannels(nodeId, channels1.values))
+      // let's clean the db and send the events
+      log.info(s"pruning shortChannelId=$shortChannelId (spent)")
       db.removeChannel(shortChannelId) // NB: this also removes channel updates
-      lostNodes.foreach(nodeId => db.removeNode(nodeId))
+      context.system.eventStream.publish(ChannelLost(shortChannelId))
+      lostNodes.foreach {
+        case nodeId =>
+          log.info(s"pruning nodeId=$nodeId (spent)")
+          db.removeNode(nodeId)
+          context.system.eventStream.publish(NodeLost(nodeId))
+      }
       stay using d.copy(nodes = d.nodes -- lostNodes, channels = d.channels - shortChannelId, updates = d.updates.filterKeys(_.id != shortChannelId))
 
-    case Event('tick_validate, d) => stay // ignored
+    case Event(TickValidate, d) => stay // ignored
 
-    case Event('tick_broadcast, d) =>
+    case Event(TickBroadcast, d) =>
       d.rebroadcast match {
         case Nil => stay using d.copy(origins = Map.empty)
         case _ =>
@@ -261,14 +266,37 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
           stay using d.copy(rebroadcast = Nil, origins = Map.empty)
       }
 
-    case Event(ExcludeChannel(desc@ChannelDesc(channelId, nodeId, _)), d) =>
+    case Event(TickPruneStaleChannels, d) =>
+      // first we select channels that we will prune
+      val staleChannels = getStaleChannels(d.channels, d.updates)
+      // then we clean up the related channel updates
+      val staleUpdates = d.updates.keys.filter(desc => staleChannels.contains(desc.id))
+      // finally we remove nodes that aren't tied to any channels anymore
+      val channels1 = d.channels -- staleChannels
+      val staleNodes = d.nodes.keys.filterNot(nodeId => hasChannels(nodeId, channels1.values))
+      // let's clean the db and send the events
+      staleChannels.foreach {
+        case shortChannelId =>
+          log.info(s"pruning shortChannelId=$shortChannelId (stale)")
+          db.removeChannel(shortChannelId) // NB: this also removes channel updates
+          context.system.eventStream.publish(ChannelLost(shortChannelId))
+      }
+      staleNodes.foreach {
+        case nodeId =>
+          log.info(s"pruning nodeId=$nodeId (stale)")
+          db.removeNode(nodeId)
+          context.system.eventStream.publish(NodeLost(nodeId))
+      }
+      stay using d.copy(nodes = d.nodes -- staleNodes, channels = channels1, updates = d.updates -- staleUpdates)
+
+    case Event(ExcludeChannel(desc@ChannelDesc(shortChannelId, nodeId, _)), d) =>
       val banDuration = nodeParams.channelExcludeDuration
-      log.info(s"excluding channelId=$channelId from nodeId=$nodeId for duration=$banDuration")
+      log.info(s"excluding shortChannelId=$shortChannelId from nodeId=$nodeId for duration=$banDuration")
       context.system.scheduler.scheduleOnce(banDuration, self, LiftChannelExclusion(desc))
       stay using d.copy(excludedChannels = d.excludedChannels + desc)
 
-    case Event(LiftChannelExclusion(desc@ChannelDesc(channelId, nodeId, _)), d) =>
-      log.info(s"reinstating channelId=$channelId from nodeId=$nodeId")
+    case Event(LiftChannelExclusion(desc@ChannelDesc(shortChannelId, nodeId, _)), d) =>
+      log.info(s"reinstating shortChannelId=$shortChannelId from nodeId=$nodeId")
       stay using d.copy(excludedChannels = d.excludedChannels - desc)
 
     case Event('nodes, d) =>
@@ -319,13 +347,6 @@ class Router(nodeParams: NodeParams, watcher: ActorRef) extends FSM[State, Data]
 
 object Router {
 
-  // TODO: temporary, required because we stored all three types of announcements in the same key-value database
-  // @formatter:off
-  def nodeKey(nodeId: BinaryData) = s"ann-node-$nodeId"
-  def channelKey(shortChannelId: Long) = s"ann-channel-$shortChannelId"
-  def channelUpdateKey(shortChannelId: Long, flags: BinaryData) = s"ann-update-$shortChannelId-$flags"
-  // @formatter:on
-
   val MAX_PARALLEL_JSONRPC_REQUESTS = 50
 
   def props(nodeParams: NodeParams, watcher: ActorRef) = Props(new Router(nodeParams, watcher))
@@ -336,7 +357,24 @@ object Router {
     if (Announcements.isNode1(u.flags)) ChannelDesc(u.shortChannelId, channel.nodeId1, channel.nodeId2) else ChannelDesc(u.shortChannelId, channel.nodeId2, channel.nodeId1)
   }
 
-  def isRelatedTo(c: ChannelAnnouncement, n: NodeAnnouncement) = n.nodeId == c.nodeId1 || n.nodeId == c.nodeId2
+  def isRelatedTo(c: ChannelAnnouncement, nodeId: PublicKey) = nodeId == c.nodeId1 || nodeId == c.nodeId2
+
+  def hasChannels(nodeId: PublicKey, channels: Iterable[ChannelAnnouncement]): Boolean = channels.exists(c => isRelatedTo(c, nodeId))
+
+  def getStaleChannels(channels: Map[Long, ChannelAnnouncement], updates: Map[ChannelDesc, ChannelUpdate]): Iterable[Long] = {
+    // BOLT 7: "nodes MAY prune channels should the timestamp of the latest channel_update be older than 2 weeks (1209600 seconds)"
+    // but we don't want to prune brand new channels for which we didn't yet receive a channel update
+    // so we consider stale a channel that:
+    // (1) is older than 2 weeks (2*7*144 = 2016 blocks)
+    //  AND
+    // (2) didn't have an update during the last 2 weeks
+    val staleThresholdSeconds = Platform.currentTime / 1000 - 1209600
+    val staleThresholdBlocks = Globals.blockCount.get() - 2016
+    val staleChannels = channels
+      .filterKeys(shortChannelId => fromShortId(shortChannelId)._1 < staleThresholdBlocks) // consider only channels older than 2 weeks
+      .filterKeys(shortChannelId => !updates.values.exists(u => u.shortChannelId == shortChannelId && u.timestamp >= staleThresholdSeconds)) // no update in the past 2 weeks
+    staleChannels.keys
+  }
 
   /**
     * This method is used after a payment failed, and we want to exclude some nodes/channels that we know are failing

eclair-core/src/main/scala/fr/acinq/eclair/router/RouterExceptions.scala

@@ -7,4 +7,5 @@ package fr.acinq.eclair.router
 class RouterException(message: String) extends RuntimeException(message)
 
 object RouteNotFound extends RouterException("Route not found")
+
 object CannotRouteToSelf extends RouterException("Cannot route to self")

eclair-core/src/main/scala/fr/acinq/eclair/router/ThrottleForwarder.scala

@@ -0,0 +1,47 @@
+package fr.acinq.eclair.router
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+
+import scala.concurrent.duration.{FiniteDuration, _}
+
+/**
+  * This actor forwards messages to another actor, but groups them and introduces
+  * delays between each groups.
+  *
+  * If A wants to send a lot of lower importance messages to B, it is useful to let
+  * higher importance messages go in the stream.
+  */
+class ThrottleForwarder(target: ActorRef, messages: Iterable[Any], chunkSize: Int, delay: FiniteDuration) extends Actor with ActorLogging {
+
+  import ThrottleForwarder.Tick
+
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  val clock = context.system.scheduler.schedule(0 second, delay, self, Tick)
+
+  log.debug(s"sending messages=${messages.size} with chunkSize=$chunkSize and delay=$delay")
+
+  override def receive = group(messages)
+
+  def group(messages: Iterable[Any]): Receive = {
+    case Tick =>
+      messages.splitAt(chunkSize) match {
+        case (Nil, _) =>
+          clock.cancel()
+          log.debug(s"sent messages=${messages.size} with chunkSize=$chunkSize and delay=$delay")
+          context stop self
+        case (chunk, rest) =>
+          chunk.foreach(target ! _)
+          context become group(rest)
+      }
+  }
+
+}
+
+object ThrottleForwarder {
+
+  def props(target: ActorRef, messages: Iterable[Any], groupSize: Int, delay: FiniteDuration) = Props(new ThrottleForwarder(target, messages, groupSize, delay))
+
+  case object Tick
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/transactions/CommitmentSpec.scala

@@ -1,6 +1,5 @@
 package fr.acinq.eclair.transactions
 
-import fr.acinq.bitcoin.BinaryData
 import fr.acinq.eclair.wire._
 
 /**
@@ -13,9 +12,9 @@ case object IN extends Direction { def opposite = OUT }
 case object OUT extends Direction { def opposite = IN }
 // @formatter:on
 
-case class Htlc(direction: Direction, add: UpdateAddHtlc, val previousChannelId: Option[BinaryData])
+case class DirectedHtlc(direction: Direction, add: UpdateAddHtlc)
 
-final case class CommitmentSpec(htlcs: Set[Htlc], feeratePerKw: Long, toLocalMsat: Long, toRemoteMsat: Long) {
+final case class CommitmentSpec(htlcs: Set[DirectedHtlc], feeratePerKw: Long, toLocalMsat: Long, toRemoteMsat: Long) {
   val totalFunds = toLocalMsat + toRemoteMsat + htlcs.toSeq.map(_.add.amountMsat).sum
 }
 
@@ -26,7 +25,7 @@ object CommitmentSpec {
   })
 
   def addHtlc(spec: CommitmentSpec, direction: Direction, update: UpdateAddHtlc): CommitmentSpec = {
-    val htlc = Htlc(direction, update, previousChannelId = None)
+    val htlc = DirectedHtlc(direction, update)
     direction match {
       case OUT => spec.copy(toLocalMsat = spec.toLocalMsat - htlc.add.amountMsat, htlcs = spec.htlcs + htlc)
       case IN => spec.copy(toRemoteMsat = spec.toRemoteMsat - htlc.add.amountMsat, htlcs = spec.htlcs + htlc)
@@ -34,20 +33,20 @@ object CommitmentSpec {
   }
 
   // OUT means we are sending an UpdateFulfillHtlc message which means that we are fulfilling an HTLC that they sent
-  def fulfillHtlc(spec: CommitmentSpec, direction: Direction, update: UpdateFulfillHtlc): CommitmentSpec = {
-    spec.htlcs.find(htlc => htlc.direction != direction && htlc.add.id == update.id) match {
+  def fulfillHtlc(spec: CommitmentSpec, direction: Direction, htlcId: Long): CommitmentSpec = {
+    spec.htlcs.find(htlc => htlc.direction != direction && htlc.add.id == htlcId) match {
       case Some(htlc) if direction == OUT => spec.copy(toLocalMsat = spec.toLocalMsat + htlc.add.amountMsat, htlcs = spec.htlcs - htlc)
       case Some(htlc) if direction == IN => spec.copy(toRemoteMsat = spec.toRemoteMsat + htlc.add.amountMsat, htlcs = spec.htlcs - htlc)
-      case None => throw new RuntimeException(s"cannot find htlc id=${update.id}")
+      case None => throw new RuntimeException(s"cannot find htlc id=${htlcId}")
     }
   }
 
   // OUT means we are sending an UpdateFailHtlc message which means that we are failing an HTLC that they sent
-  def failHtlc(spec: CommitmentSpec, direction: Direction, update: UpdateFailHtlc): CommitmentSpec = {
-    spec.htlcs.find(htlc => htlc.direction != direction && htlc.add.id == update.id) match {
+  def failHtlc(spec: CommitmentSpec, direction: Direction, htlcId: Long): CommitmentSpec = {
+    spec.htlcs.find(htlc => htlc.direction != direction && htlc.add.id == htlcId) match {
       case Some(htlc) if direction == OUT => spec.copy(toRemoteMsat = spec.toRemoteMsat + htlc.add.amountMsat, htlcs = spec.htlcs - htlc)
       case Some(htlc) if direction == IN => spec.copy(toLocalMsat = spec.toLocalMsat + htlc.add.amountMsat, htlcs = spec.htlcs - htlc)
-      case None => throw new RuntimeException(s"cannot find htlc id=${update.id}")
+      case None => throw new RuntimeException(s"cannot find htlc id=${htlcId}")
     }
   }
 
@@ -61,13 +60,15 @@ object CommitmentSpec {
       case (spec, _) => spec
     }
     val spec3 = localChanges.foldLeft(spec2) {
-      case (spec, u: UpdateFulfillHtlc) => fulfillHtlc(spec, OUT, u)
-      case (spec, u: UpdateFailHtlc) => failHtlc(spec, OUT, u)
+      case (spec, u: UpdateFulfillHtlc) => fulfillHtlc(spec, OUT, u.id)
+      case (spec, u: UpdateFailHtlc) => failHtlc(spec, OUT, u.id)
+      case (spec, u: UpdateFailMalformedHtlc) => failHtlc(spec, OUT, u.id)
       case (spec, _) => spec
     }
     val spec4 = remoteChanges.foldLeft(spec3) {
-      case (spec, u: UpdateFulfillHtlc) => fulfillHtlc(spec, IN, u)
-      case (spec, u: UpdateFailHtlc) => failHtlc(spec, IN, u)
+      case (spec, u: UpdateFulfillHtlc) => fulfillHtlc(spec, IN, u.id)
+      case (spec, u: UpdateFailHtlc) => failHtlc(spec, IN, u.id)
+      case (spec, u: UpdateFailMalformedHtlc) => failHtlc(spec, IN, u.id)
       case (spec, _) => spec
     }
     val spec5 = (localChanges ++ remoteChanges).foldLeft(spec4) {

eclair-core/src/main/scala/fr/acinq/eclair/transactions/Scripts.scala

@@ -1,8 +1,8 @@
 package fr.acinq.eclair.transactions
 
-import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey, ripemd160}
+import fr.acinq.bitcoin.Crypto.{PublicKey, ripemd160}
 import fr.acinq.bitcoin.Script._
-import fr.acinq.bitcoin.{BinaryData, Crypto, LexicographicalOrdering, LockTimeThreshold, OP_0, OP_1, OP_1NEGATE, OP_2, OP_2DROP, OP_ADD, OP_CHECKLOCKTIMEVERIFY, OP_CHECKMULTISIG, OP_CHECKSEQUENCEVERIFY, OP_CHECKSIG, OP_DROP, OP_DUP, OP_ELSE, OP_ENDIF, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_IF, OP_NOTIF, OP_PUSHDATA, OP_SIZE, OP_SWAP, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptElt, ScriptWitness, Transaction, TxIn, TxOut}
+import fr.acinq.bitcoin.{BinaryData, LexicographicalOrdering, LockTimeThreshold, OP_0, OP_1, OP_1NEGATE, OP_2, OP_2DROP, OP_ADD, OP_CHECKLOCKTIMEVERIFY, OP_CHECKMULTISIG, OP_CHECKSEQUENCEVERIFY, OP_CHECKSIG, OP_DROP, OP_DUP, OP_ELSE, OP_ENDIF, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_IF, OP_NOTIF, OP_PUSHDATA, OP_SIZE, OP_SWAP, Satoshi, Script, ScriptElt, ScriptWitness, Transaction, TxIn}
 
 /**
   * Created by PM on 02/12/2016.
@@ -156,13 +156,13 @@ object Scripts {
     else tx.txIn.map(_.sequence).map(sequenceToBlockHeight).max
   }
 
-  def toLocalDelayed(revocationPubkey: PublicKey, toSelfDelay: Int, localDelayedPubkey: PublicKey) = {
+  def toLocalDelayed(revocationPubkey: PublicKey, toSelfDelay: Int, localDelayedPaymentPubkey: PublicKey) = {
     // @formatter:off
     OP_IF ::
       OP_PUSHDATA(revocationPubkey) ::
     OP_ELSE ::
       encodeNumber(toSelfDelay) :: OP_CHECKSEQUENCEVERIFY :: OP_DROP ::
-      OP_PUSHDATA(localDelayedPubkey) ::
+      OP_PUSHDATA(localDelayedPaymentPubkey) ::
     OP_ENDIF ::
     OP_CHECKSIG :: Nil
     // @formatter:on
@@ -181,17 +181,17 @@ object Scripts {
   def witnessToLocalDelayedWithRevocationSig(revocationSig: BinaryData, toLocalScript: BinaryData) =
     ScriptWitness(revocationSig :: BinaryData("01") :: toLocalScript :: Nil)
 
-  def htlcOffered(localPubkey: PublicKey, remotePubkey: PublicKey, revocationPubKey: PublicKey, paymentHash: BinaryData): Seq[ScriptElt] = {
+  def htlcOffered(localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, revocationPubKey: PublicKey, paymentHash: BinaryData): Seq[ScriptElt] = {
     // @formatter:off
     // To you with revocation key
     OP_DUP :: OP_HASH160 :: OP_PUSHDATA(revocationPubKey.hash160) :: OP_EQUAL ::
     OP_IF ::
         OP_CHECKSIG ::
     OP_ELSE ::
-        OP_PUSHDATA(remotePubkey) :: OP_SWAP  :: OP_SIZE :: encodeNumber(32) :: OP_EQUAL ::
+        OP_PUSHDATA(remoteHtlcPubkey) :: OP_SWAP  :: OP_SIZE :: encodeNumber(32) :: OP_EQUAL ::
         OP_NOTIF ::
             // To me via HTLC-timeout transaction (timelocked).
-            OP_DROP :: OP_2 :: OP_SWAP :: OP_PUSHDATA(localPubkey) :: OP_2 :: OP_CHECKMULTISIG ::
+            OP_DROP :: OP_2 :: OP_SWAP :: OP_PUSHDATA(localHtlcPubkey) :: OP_2 :: OP_CHECKMULTISIG ::
         OP_ELSE ::
             OP_HASH160 :: OP_PUSHDATA(paymentHash) :: OP_EQUALVERIFY ::
             OP_CHECKSIG ::
@@ -213,18 +213,18 @@ object Scripts {
   def witnessClaimHtlcSuccessFromCommitTx(localSig: BinaryData, paymentPreimage: BinaryData, htlcOfferedScript: BinaryData) =
     ScriptWitness(localSig :: paymentPreimage :: htlcOfferedScript :: Nil)
 
-  def htlcReceived(localKey: PublicKey, remotePubkey: PublicKey, revocationPubKey: PublicKey, paymentHash: BinaryData, lockTime: Long) = {
+  def htlcReceived(localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, revocationPubKey: PublicKey, paymentHash: BinaryData, lockTime: Long) = {
     // @formatter:off
     // To you with revocation key
     OP_DUP :: OP_HASH160 :: OP_PUSHDATA(revocationPubKey.hash160) :: OP_EQUAL ::
     OP_IF ::
         OP_CHECKSIG ::
     OP_ELSE ::
-        OP_PUSHDATA(remotePubkey) :: OP_SWAP :: OP_SIZE :: encodeNumber(32) :: OP_EQUAL ::
+        OP_PUSHDATA(remoteHtlcPubkey) :: OP_SWAP :: OP_SIZE :: encodeNumber(32) :: OP_EQUAL ::
         OP_IF ::
             // To me via HTLC-success transaction.
             OP_HASH160 :: OP_PUSHDATA(paymentHash) :: OP_EQUALVERIFY ::
-            OP_2 :: OP_SWAP :: OP_PUSHDATA(localKey) :: OP_2 :: OP_CHECKMULTISIG ::
+            OP_2 :: OP_SWAP :: OP_PUSHDATA(localHtlcPubkey) :: OP_2 :: OP_CHECKMULTISIG ::
         OP_ELSE ::
             // To you after timeout.
             OP_DROP :: encodeNumber(lockTime) :: OP_CHECKLOCKTIMEVERIFY :: OP_DROP ::

eclair-core/src/main/scala/fr/acinq/eclair/transactions/Transactions.scala

@@ -73,7 +73,7 @@ object Transactions {
 
   def weight2fee(feeratePerKw: Long, weight: Int) = Satoshi((feeratePerKw * weight) / 1000)
 
-  def trimOfferedHtlcs(dustLimit: Satoshi, spec: CommitmentSpec): Seq[Htlc] = {
+  def trimOfferedHtlcs(dustLimit: Satoshi, spec: CommitmentSpec): Seq[DirectedHtlc] = {
     val htlcTimeoutFee = weight2fee(spec.feeratePerKw, htlcTimeoutWeight)
     spec.htlcs
       .filter(_.direction == OUT)
@@ -81,7 +81,7 @@ object Transactions {
       .toSeq
   }
 
-  def trimReceivedHtlcs(dustLimit: Satoshi, spec: CommitmentSpec): Seq[Htlc] = {
+  def trimReceivedHtlcs(dustLimit: Satoshi, spec: CommitmentSpec): Seq[DirectedHtlc] = {
     val htlcSuccessFee = weight2fee(spec.feeratePerKw, htlcSuccessWeight)
     spec.htlcs
       .filter(_.direction == IN)
@@ -142,7 +142,7 @@ object Transactions {
 
   def decodeTxNumber(sequence: Long, locktime: Long) = ((sequence & 0xffffffL) << 24) + (locktime & 0xffffffL)
 
-  def makeCommitTx(commitTxInput: InputInfo, commitTxNumber: Long, localPaymentBasePoint: Point, remotePaymentBasePoint: Point, localIsFunder: Boolean, localDustLimit: Satoshi, localPubKey: PublicKey, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPubkey: PublicKey, remotePubkey: PublicKey, spec: CommitmentSpec): CommitTx = {
+  def makeCommitTx(commitTxInput: InputInfo, commitTxNumber: Long, localPaymentBasePoint: Point, remotePaymentBasePoint: Point, localIsFunder: Boolean, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPaymentPubkey: PublicKey, remotePaymentPubkey: PublicKey, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, spec: CommitmentSpec): CommitTx = {
 
     val commitFee = commitTxFee(localDustLimit, spec)
 
@@ -151,13 +151,13 @@ object Transactions {
       case false => (millisatoshi2satoshi(MilliSatoshi(spec.toLocalMsat)), millisatoshi2satoshi(MilliSatoshi(spec.toRemoteMsat)) - commitFee)
     } // NB: we don't care if values are < 0, they will be trimmed if they are < dust limit anyway
 
-    val toLocalDelayedOutput_opt = if (toLocalAmount >= localDustLimit) Some(TxOut(toLocalAmount, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPubkey)))) else None
-    val toRemoteOutput_opt = if (toRemoteAmount >= localDustLimit) Some(TxOut(toRemoteAmount, pay2wpkh(remotePubkey))) else None
+    val toLocalDelayedOutput_opt = if (toLocalAmount >= localDustLimit) Some(TxOut(toLocalAmount, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey)))) else None
+    val toRemoteOutput_opt = if (toRemoteAmount >= localDustLimit) Some(TxOut(toRemoteAmount, pay2wpkh(remotePaymentPubkey))) else None
 
     val htlcOfferedOutputs = trimOfferedHtlcs(localDustLimit, spec)
-      .map(htlc => TxOut(MilliSatoshi(htlc.add.amountMsat), pay2wsh(htlcOffered(localPubKey, remotePubkey, localRevocationPubkey, ripemd160(htlc.add.paymentHash)))))
+      .map(htlc => TxOut(MilliSatoshi(htlc.add.amountMsat), pay2wsh(htlcOffered(localHtlcPubkey, remoteHtlcPubkey, localRevocationPubkey, ripemd160(htlc.add.paymentHash)))))
     val htlcReceivedOutputs = trimReceivedHtlcs(localDustLimit, spec)
-      .map(htlc => TxOut(MilliSatoshi(htlc.add.amountMsat), pay2wsh(htlcReceived(localPubKey, remotePubkey, localRevocationPubkey, ripemd160(htlc.add.paymentHash), htlc.add.expiry))))
+      .map(htlc => TxOut(MilliSatoshi(htlc.add.amountMsat), pay2wsh(htlcReceived(localHtlcPubkey, remoteHtlcPubkey, localRevocationPubkey, ripemd160(htlc.add.paymentHash), htlc.add.expiry))))
 
     val txnumber = obscuredCommitTxNumber(commitTxNumber, localIsFunder, localPaymentBasePoint, remotePaymentBasePoint)
     val (sequence, locktime) = encodeTxNumber(txnumber)
@@ -170,113 +170,127 @@ object Transactions {
     CommitTx(commitTxInput, LexicographicalOrdering.sort(tx))
   }
 
-  def makeHtlcTimeoutTx(commitTx: Transaction, localRevocationPubkey: PublicKey, toLocalDelay: Int, localPubKey: PublicKey, localDelayedPubkey: PublicKey, remotePubkey: PublicKey, feeratePerKw: Long, htlc: UpdateAddHtlc): HtlcTimeoutTx = {
+  def makeHtlcTimeoutTx(commitTx: Transaction, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPaymentPubkey: PublicKey, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, feeratePerKw: Long, htlc: UpdateAddHtlc): HtlcTimeoutTx = {
     val fee = weight2fee(feeratePerKw, htlcTimeoutWeight)
-    val redeemScript = htlcOffered(localPubKey, remotePubkey, localRevocationPubkey, ripemd160(htlc.paymentHash))
+    val redeemScript = htlcOffered(localHtlcPubkey, remoteHtlcPubkey, localRevocationPubkey, ripemd160(htlc.paymentHash))
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(commitTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
+    val amount = MilliSatoshi(htlc.amountMsat) - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     val input = InputInfo(OutPoint(commitTx, outputIndex), commitTx.txOut(outputIndex), write(redeemScript))
     HtlcTimeoutTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0) :: Nil,
-      txOut = TxOut(MilliSatoshi(htlc.amountMsat) - fee, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPubkey))) :: Nil,
+      txOut = TxOut(amount, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey))) :: Nil,
       lockTime = htlc.expiry))
   }
 
-  def makeHtlcSuccessTx(commitTx: Transaction, localRevocationPubkey: PublicKey, toLocalDelay: Int, localPubkey: PublicKey, localDelayedPubkey: PublicKey, remotePubkey: PublicKey, feeratePerKw: Long, htlc: UpdateAddHtlc): HtlcSuccessTx = {
+  def makeHtlcSuccessTx(commitTx: Transaction, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPaymentPubkey: PublicKey, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, feeratePerKw: Long, htlc: UpdateAddHtlc): HtlcSuccessTx = {
     val fee = weight2fee(feeratePerKw, htlcSuccessWeight)
-    val redeemScript = htlcReceived(localPubkey, remotePubkey, localRevocationPubkey, ripemd160(htlc.paymentHash), htlc.expiry)
+    val redeemScript = htlcReceived(localHtlcPubkey, remoteHtlcPubkey, localRevocationPubkey, ripemd160(htlc.paymentHash), htlc.expiry)
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(commitTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
+    val amount = MilliSatoshi(htlc.amountMsat) - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     val input = InputInfo(OutPoint(commitTx, outputIndex), commitTx.txOut(outputIndex), write(redeemScript))
     HtlcSuccessTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0) :: Nil,
-      txOut = TxOut(MilliSatoshi(htlc.amountMsat) - fee, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPubkey))) :: Nil,
+      txOut = TxOut(amount, pay2wsh(toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey))) :: Nil,
       lockTime = 0), htlc.paymentHash)
   }
 
-  def makeHtlcTxs(commitTx: Transaction, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localPubkey: PublicKey, localDelayedPubkey: PublicKey, remotePubkey: PublicKey, spec: CommitmentSpec): (Seq[HtlcTimeoutTx], Seq[HtlcSuccessTx]) = {
+  def makeHtlcTxs(commitTx: Transaction, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPaymentPubkey: PublicKey, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, spec: CommitmentSpec): (Seq[HtlcTimeoutTx], Seq[HtlcSuccessTx]) = {
     val htlcTimeoutTxs = trimOfferedHtlcs(localDustLimit, spec)
-      .map(htlc => makeHtlcTimeoutTx(commitTx, localRevocationPubkey, toLocalDelay, localPubkey, localDelayedPubkey, remotePubkey, spec.feeratePerKw, htlc.add))
+      .map(htlc => makeHtlcTimeoutTx(commitTx, localDustLimit, localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey, localHtlcPubkey, remoteHtlcPubkey, spec.feeratePerKw, htlc.add))
     val htlcSuccessTxs = trimReceivedHtlcs(localDustLimit, spec)
-      .map(htlc => makeHtlcSuccessTx(commitTx, localRevocationPubkey, toLocalDelay, localPubkey, localDelayedPubkey, remotePubkey, spec.feeratePerKw, htlc.add))
+      .map(htlc => makeHtlcSuccessTx(commitTx, localDustLimit, localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey, localHtlcPubkey, remoteHtlcPubkey, spec.feeratePerKw, htlc.add))
     (htlcTimeoutTxs, htlcSuccessTxs)
   }
 
-  def makeClaimHtlcSuccessTx(commitTx: Transaction, localPubkey: PublicKey, remotePubkey: PublicKey, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, htlc: UpdateAddHtlc, feeratePerKw: Long): ClaimHtlcSuccessTx = {
+  def makeClaimHtlcSuccessTx(commitTx: Transaction, localDustLimit: Satoshi, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, htlc: UpdateAddHtlc, feeratePerKw: Long): ClaimHtlcSuccessTx = {
     val fee = weight2fee(feeratePerKw, claimHtlcSuccessWeight)
-    val redeemScript = htlcOffered(remotePubkey, localPubkey, remoteRevocationPubkey, ripemd160(htlc.paymentHash))
+    val redeemScript = htlcOffered(remoteHtlcPubkey, localHtlcPubkey, remoteRevocationPubkey, ripemd160(htlc.paymentHash))
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(commitTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
     val input = InputInfo(OutPoint(commitTx, outputIndex), commitTx.txOut(outputIndex), write(redeemScript))
+    val amount = input.txOut.amount - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     ClaimHtlcSuccessTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0xffffffffL) :: Nil,
-      txOut = TxOut(input.txOut.amount - fee, localFinalScriptPubKey) :: Nil,
+      txOut = TxOut(amount, localFinalScriptPubKey) :: Nil,
       lockTime = 0))
   }
 
-  def makeClaimHtlcTimeoutTx(commitTx: Transaction, localPubkey: PublicKey, remotePubkey: PublicKey, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, htlc: UpdateAddHtlc, feeratePerKw: Long): ClaimHtlcTimeoutTx = {
+  def makeClaimHtlcTimeoutTx(commitTx: Transaction, localDustLimit: Satoshi, localHtlcPubkey: PublicKey, remoteHtlcPubkey: PublicKey, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, htlc: UpdateAddHtlc, feeratePerKw: Long): ClaimHtlcTimeoutTx = {
     val fee = weight2fee(feeratePerKw, claimHtlcTimeoutWeight)
-    val redeemScript = htlcReceived(remotePubkey, localPubkey, remoteRevocationPubkey, ripemd160(htlc.paymentHash), htlc.expiry)
+    val redeemScript = htlcReceived(remoteHtlcPubkey, localHtlcPubkey, remoteRevocationPubkey, ripemd160(htlc.paymentHash), htlc.expiry)
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(commitTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
     val input = InputInfo(OutPoint(commitTx, outputIndex), commitTx.txOut(outputIndex), write(redeemScript))
+    val amount = input.txOut.amount - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     ClaimHtlcTimeoutTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0x00000000L) :: Nil,
-      txOut = TxOut(input.txOut.amount - fee, localFinalScriptPubKey) :: Nil,
+      txOut = TxOut(amount, localFinalScriptPubKey) :: Nil,
       lockTime = htlc.expiry))
   }
 
-  def makeClaimP2WPKHOutputTx(delayedOutputTx: Transaction, localPubkey: PublicKey, localFinalScriptPubKey: BinaryData, feeratePerKw: Long): ClaimP2WPKHOutputTx = {
+  def makeClaimP2WPKHOutputTx(delayedOutputTx: Transaction, localDustLimit: Satoshi, localPaymentPubkey: PublicKey, localFinalScriptPubKey: BinaryData, feeratePerKw: Long): ClaimP2WPKHOutputTx = {
     val fee = weight2fee(feeratePerKw, claimP2WPKHOutputWeight)
-    val redeemScript = Script.pay2pkh(localPubkey)
-    val pubkeyScript = write(pay2wpkh(localPubkey))
+    val redeemScript = Script.pay2pkh(localPaymentPubkey)
+    val pubkeyScript = write(pay2wpkh(localPaymentPubkey))
     val outputIndex = findPubKeyScriptIndex(delayedOutputTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
     val input = InputInfo(OutPoint(delayedOutputTx, outputIndex), delayedOutputTx.txOut(outputIndex), write(redeemScript))
+    val amount = input.txOut.amount - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     ClaimP2WPKHOutputTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0x00000000L) :: Nil,
-      txOut = TxOut(input.txOut.amount - fee, localFinalScriptPubKey) :: Nil,
+      txOut = TxOut(amount, localFinalScriptPubKey) :: Nil,
       lockTime = 0))
   }
 
-  def makeClaimDelayedOutputTx(delayedOutputTx: Transaction, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPubkey: PublicKey, localFinalScriptPubKey: BinaryData, feeratePerKw: Long): ClaimDelayedOutputTx = {
+  def makeClaimDelayedOutputTx(delayedOutputTx: Transaction, localDustLimit: Satoshi, localRevocationPubkey: PublicKey, toLocalDelay: Int, localDelayedPaymentPubkey: PublicKey, localFinalScriptPubKey: BinaryData, feeratePerKw: Long): ClaimDelayedOutputTx = {
     val fee = weight2fee(feeratePerKw, claimHtlcDelayedWeight)
-    val redeemScript = toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPubkey)
+    val redeemScript = toLocalDelayed(localRevocationPubkey, toLocalDelay, localDelayedPaymentPubkey)
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(delayedOutputTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
     val input = InputInfo(OutPoint(delayedOutputTx, outputIndex), delayedOutputTx.txOut(outputIndex), write(redeemScript))
+    val amount = input.txOut.amount - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     ClaimDelayedOutputTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, toLocalDelay) :: Nil,
-      txOut = TxOut(input.txOut.amount - fee, localFinalScriptPubKey) :: Nil,
+      txOut = TxOut(amount, localFinalScriptPubKey) :: Nil,
       lockTime = 0))
   }
 
-  def makeMainPenaltyTx(commitTx: Transaction, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, toRemoteDelay: Int, remoteDelayedPubkey: PublicKey, feeratePerKw: Long): MainPenaltyTx = {
+  def makeMainPenaltyTx(commitTx: Transaction, localDustLimit: Satoshi, remoteRevocationPubkey: PublicKey, localFinalScriptPubKey: BinaryData, toRemoteDelay: Int, remoteDelayedPaymentPubkey: PublicKey, feeratePerKw: Long): MainPenaltyTx = {
     val fee = weight2fee(feeratePerKw, mainPenaltyWeight)
-    val redeemScript = toLocalDelayed(remoteRevocationPubkey, toRemoteDelay, remoteDelayedPubkey)
+    val redeemScript = toLocalDelayed(remoteRevocationPubkey, toRemoteDelay, remoteDelayedPaymentPubkey)
     val pubkeyScript = write(pay2wsh(redeemScript))
     val outputIndex = findPubKeyScriptIndex(commitTx, pubkeyScript)
-    require(outputIndex >= 0, "output not found")
+    require(outputIndex >= 0, "output not found (was trimmed?)")
     val input = InputInfo(OutPoint(commitTx, outputIndex), commitTx.txOut(outputIndex), write(redeemScript))
+    val amount = input.txOut.amount - fee
+    require(amount >= localDustLimit, "amount lesser than dust limit")
     MainPenaltyTx(input, Transaction(
       version = 2,
       txIn = TxIn(input.outPoint, Array.emptyByteArray, 0xffffffffL) :: Nil,
-      txOut = TxOut(input.txOut.amount - fee, localFinalScriptPubKey) :: Nil,
+      txOut = TxOut(amount, localFinalScriptPubKey) :: Nil,
       lockTime = 0))
   }
 
-  def makeHtlcPenaltyTx(commitTx: Transaction): HtlcPenaltyTx = ???
+  def makeHtlcPenaltyTx(commitTx: Transaction, localDustLimit: Satoshi): HtlcPenaltyTx = ???
 
   def makeClosingTx(commitTxInput: InputInfo, localScriptPubKey: BinaryData, remoteScriptPubKey: BinaryData, localIsFunder: Boolean, dustLimit: Satoshi, closingFee: Satoshi, spec: CommitmentSpec): ClosingTx = {
     require(spec.htlcs.size == 0, "there shouldn't be any pending htlcs")
@@ -348,8 +362,8 @@ object Transactions {
     claimHtlcTimeoutTx.copy(tx = claimHtlcTimeoutTx.tx.updateWitness(0, witness))
   }
 
-  def addSigs(claimP2WPKHOutputTx: ClaimP2WPKHOutputTx, localPubkey: BinaryData, localSig: BinaryData): ClaimP2WPKHOutputTx = {
-    val witness = ScriptWitness(Seq(localSig, localPubkey))
+  def addSigs(claimP2WPKHOutputTx: ClaimP2WPKHOutputTx, localPaymentPubkey: BinaryData, localSig: BinaryData): ClaimP2WPKHOutputTx = {
+    val witness = ScriptWitness(Seq(localSig, localPaymentPubkey))
     claimP2WPKHOutputTx.copy(tx = claimP2WPKHOutputTx.tx.updateWitness(0, witness))
   }
 

eclair-core/src/main/scala/fr/acinq/eclair/wire/ChannelCodecs.scala

@@ -1,8 +1,9 @@
 package fr.acinq.eclair.wire
 
-import fr.acinq.bitcoin.{OutPoint, Transaction, TxOut}
+import fr.acinq.bitcoin.{BinaryData, OutPoint, Transaction, TxOut}
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.ShaChain
+import fr.acinq.eclair.payment.{Local, Origin, Relayed}
 import fr.acinq.eclair.transactions.Transactions._
 import fr.acinq.eclair.transactions._
 import fr.acinq.eclair.wire.LightningMessageCodecs._
@@ -25,8 +26,9 @@ object ChannelCodecs {
       ("maxAcceptedHtlcs" | uint16) ::
       ("fundingPrivKey" | privateKey) ::
       ("revocationSecret" | scalar) ::
-      ("paymentKey" | privateKey) ::
+      ("paymentKey" | scalar) ::
       ("delayedPaymentKey" | scalar) ::
+      ("htlcKey" | scalar) ::
       ("defaultFinalScriptPubKey" | varsizebinarydata) ::
       ("shaSeed" | varsizebinarydata) ::
       ("isFunder" | bool) ::
@@ -45,6 +47,7 @@ object ChannelCodecs {
       ("revocationBasepoint" | point) ::
       ("paymentBasepoint" | point) ::
       ("delayedPaymentBasepoint" | point) ::
+      ("htlcBasepoint" | point) ::
       ("globalFeatures" | varsizebinarydata) ::
       ("localFeatures" | varsizebinarydata)).as[RemoteParams]
 
@@ -53,10 +56,9 @@ object ChannelCodecs {
     (wire: BitVector) => bool.decode(wire).map(_.map(b => if (b) IN else OUT))
   )
 
-  val htlcCodec: Codec[Htlc] = (
+  val htlcCodec: Codec[DirectedHtlc] = (
     ("direction" | directionCodec) ::
-      ("add" | updateAddHtlcCodec) ::
-      ("previousChannelId" | optional(bool, varsizebinarydata))).as[Htlc]
+      ("add" | updateAddHtlcCodec)).as[DirectedHtlc]
 
   def setCodec[T](codec: Codec[T]): Codec[Set[T]] = Codec[Set[T]](
     (elems: Set[T]) => listOfN(uint16, codec).encode(elems.toList),
@@ -130,6 +132,23 @@ object ChannelCodecs {
       ("sentAfterLocalCommitIndex" | uint64) ::
       ("reSignAsap" | bool)).as[WaitingForRevocation]
 
+  val relayedCodec: Codec[Relayed] = (
+    ("originChannelId" | binarydata(32)) ::
+      ("originHtlcId" | int64) ::
+      ("amountMsatIn" | uint64) ::
+      ("amountMsatOut" | uint64)).as[Relayed]
+
+  val originCodec: Codec[Origin] = discriminated[Origin].by(uint16)
+    .typecase(0x01, provide(Local(None)))
+    .typecase(0x02, relayedCodec)
+
+  val originsListCodec: Codec[List[(Long, Origin)]] = listOfN(uint16, int64 ~ originCodec)
+
+  val originsMapCodec: Codec[Map[Long, Origin]] = Codec[Map[Long, Origin]](
+    (map: Map[Long, Origin]) => originsListCodec.encode(map.toList),
+    (wire: BitVector) => originsListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
   val commitmentsCodec: Codec[Commitments] = (
     ("localParams" | localParamsCodec) ::
       ("remoteParams" | remoteParamsCodec) ::
@@ -140,6 +159,7 @@ object ChannelCodecs {
       ("remoteChanges" | remoteChangesCodec) ::
       ("localNextHtlcId" | uint64) ::
       ("remoteNextHtlcId" | uint64) ::
+      ("originChannels" | originsMapCodec) ::
       ("remoteNextCommitInfo" | either(bool, waitingForRevocationCodec, point)) ::
       ("commitInput" | inputInfoCodec) ::
       ("remotePerCommitmentSecrets" | ShaChain.shaChainCodec) ::
@@ -150,13 +170,15 @@ object ChannelCodecs {
       ("claimMainDelayedOutputTx" | optional(bool, txCodec)) ::
       ("htlcSuccessTxs" | listOfN(uint16, txCodec)) ::
       ("htlcTimeoutTxs" | listOfN(uint16, txCodec)) ::
-      ("claimHtlcDelayedTx" | listOfN(uint16, txCodec))).as[LocalCommitPublished]
+      ("claimHtlcDelayedTx" | listOfN(uint16, txCodec)) ::
+      ("spent" | provide(Map.empty[OutPoint, BinaryData]))).as[LocalCommitPublished]
 
   val remoteCommitPublishedCodec: Codec[RemoteCommitPublished] = (
     ("commitTx" | txCodec) ::
       ("claimMainOutputTx" | optional(bool, txCodec)) ::
       ("claimHtlcSuccessTxs" | listOfN(uint16, txCodec)) ::
-      ("claimHtlcTimeoutTxs" | listOfN(uint16, txCodec))).as[RemoteCommitPublished]
+      ("claimHtlcTimeoutTxs" | listOfN(uint16, txCodec)) ::
+      ("spent" | provide(Map.empty[OutPoint, BinaryData]))).as[RemoteCommitPublished]
 
   val revokedCommitPublishedCodec: Codec[RevokedCommitPublished] = (
     ("commitTx" | txCodec) ::
@@ -164,7 +186,8 @@ object ChannelCodecs {
       ("mainPenaltyTx" | optional(bool, txCodec)) ::
       ("claimHtlcTimeoutTxs" | listOfN(uint16, txCodec)) ::
       ("htlcTimeoutTxs" | listOfN(uint16, txCodec)) ::
-      ("htlcPenaltyTxs" | listOfN(uint16, txCodec))).as[RevokedCommitPublished]
+      ("htlcPenaltyTxs" | listOfN(uint16, txCodec)) ::
+      ("spent" | provide(Map.empty[OutPoint, BinaryData]))).as[RevokedCommitPublished]
 
   val DATA_WAIT_FOR_FUNDING_CONFIRMED_Codec: Codec[DATA_WAIT_FOR_FUNDING_CONFIRMED] = (
     ("commitments" | commitmentsCodec) ::

eclair-core/src/main/scala/fr/acinq/eclair/wire/FailureMessage.scala

@@ -1,7 +1,7 @@
 package fr.acinq.eclair.wire
 
 import fr.acinq.bitcoin.BinaryData
-import fr.acinq.eclair.wire.LightningMessageCodecs.{binarydata, uint64, channelUpdateCodec}
+import fr.acinq.eclair.wire.LightningMessageCodecs.{binarydata, channelUpdateCodec, uint64}
 import scodec.Codec
 import scodec.codecs._
 

eclair-core/src/main/scala/fr/acinq/eclair/wire/FixedSizeStrictCodec.scala

@@ -47,9 +47,13 @@ object FixedSizeStrictCodec {
     */
   def bytesStrict(size: Int): Codec[ByteVector] = new Codec[ByteVector] {
     private val codec = new FixedSizeStrictCodec(size * 8L, codecs.bits).xmap[ByteVector](_.toByteVector, _.toBitVector)
+
     def sizeBound = codec.sizeBound
+
     def encode(b: ByteVector) = codec.encode(b)
+
     def decode(b: BitVector) = codec.decode(b)
+
     override def toString = s"bytesStrict($size)"
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/wire/LightningMessageCodecs.scala

@@ -6,8 +6,8 @@ import java.net.{Inet4Address, Inet6Address, InetAddress, InetSocketAddress}
 import fr.acinq.bitcoin.Crypto.{Point, PrivateKey, PublicKey, Scalar}
 import fr.acinq.bitcoin.{BinaryData, Crypto}
 import fr.acinq.eclair.crypto.{Generators, Sphinx}
-import fr.acinq.eclair.{UInt64, wire}
 import fr.acinq.eclair.wire.FixedSizeStrictCodec.bytesStrict
+import fr.acinq.eclair.{UInt64, wire}
 import scodec.bits.{BitVector, ByteVector}
 import scodec.codecs._
 import scodec.{Attempt, Codec, Err}
@@ -134,6 +134,7 @@ object LightningMessageCodecs {
       ("revocationBasepoint" | point) ::
       ("paymentBasepoint" | point) ::
       ("delayedPaymentBasepoint" | point) ::
+      ("htlcBasepoint" | point) ::
       ("firstPerCommitmentPoint" | point) ::
       ("channelFlags" | byte)).as[OpenChannel]
 
@@ -150,6 +151,7 @@ object LightningMessageCodecs {
       ("revocationBasepoint" | point) ::
       ("paymentBasepoint" | point) ::
       ("delayedPaymentBasepoint" | point) ::
+      ("htlcBasepoint" | point) ::
       ("firstPerCommitmentPoint" | point)).as[AcceptChannel]
 
   val fundingCreatedCodec: Codec[FundingCreated] = (

eclair-core/src/main/scala/fr/acinq/eclair/wire/LightningMessageTypes.scala

@@ -32,9 +32,9 @@ case class Ping(pongLength: Int, data: BinaryData) extends SetupMessage
 case class Pong(data: BinaryData) extends SetupMessage
 
 case class ChannelReestablish(
-                             channelId: BinaryData,
-                             nextLocalCommitmentNumber: Long,
-                             nextRemoteRevocationNumber: Long) extends ChannelMessage with HasChannelId
+                               channelId: BinaryData,
+                               nextLocalCommitmentNumber: Long,
+                               nextRemoteRevocationNumber: Long) extends ChannelMessage with HasChannelId
 
 case class OpenChannel(chainHash: BinaryData,
                        temporaryChannelId: BinaryData,
@@ -51,6 +51,7 @@ case class OpenChannel(chainHash: BinaryData,
                        revocationBasepoint: Point,
                        paymentBasepoint: Point,
                        delayedPaymentBasepoint: Point,
+                       htlcBasepoint: Point,
                        firstPerCommitmentPoint: Point,
                        channelFlags: Byte) extends ChannelMessage with HasTemporaryChannelId
 
@@ -66,6 +67,7 @@ case class AcceptChannel(temporaryChannelId: BinaryData,
                          revocationBasepoint: Point,
                          paymentBasepoint: Point,
                          delayedPaymentBasepoint: Point,
+                         htlcBasepoint: Point,
                          firstPerCommitmentPoint: Point) extends ChannelMessage with HasTemporaryChannelId
 
 case class FundingCreated(temporaryChannelId: BinaryData,

eclair-core/src/test/java/fr/acinq/eclair/crypto/Curve25519.java

@@ -27,13 +27,13 @@ import java.util.Arrays;
 
 /**
  * Implementation of the Curve25519 elliptic curve algorithm.
- *
+ * <p>
  * This implementation is based on that from arduinolibs:
  * https://github.com/rweather/arduinolibs
- *
+ * <p>
  * Differences in this version are due to using 26-bit limbs for the
  * representation instead of the 8/16/32-bit limbs in the original.
- *
+ * <p>
  * References: http://cr.yp.to/ecdh.html, RFC 7748
  */
 public final class Curve25519 {
@@ -61,25 +61,24 @@ public final class Curve25519 {
     /**
      * Constructs the temporary state holder for Curve25519 evaluation.
      */
-    private Curve25519()
-    {
+    private Curve25519() {
         // Allocate memory for all of the temporary variables we will need.
-        x_1 = new int [NUM_LIMBS_255BIT];
-        x_2 = new int [NUM_LIMBS_255BIT];
-        x_3 = new int [NUM_LIMBS_255BIT];
-        z_2 = new int [NUM_LIMBS_255BIT];
-        z_3 = new int [NUM_LIMBS_255BIT];
-        A = new int [NUM_LIMBS_255BIT];
-        B = new int [NUM_LIMBS_255BIT];
-        C = new int [NUM_LIMBS_255BIT];
-        D = new int [NUM_LIMBS_255BIT];
-        E = new int [NUM_LIMBS_255BIT];
-        AA = new int [NUM_LIMBS_255BIT];
-        BB = new int [NUM_LIMBS_255BIT];
-        DA = new int [NUM_LIMBS_255BIT];
-        CB = new int [NUM_LIMBS_255BIT];
-        t1 = new long [NUM_LIMBS_510BIT];
-        t2 = new int [NUM_LIMBS_510BIT];
+        x_1 = new int[NUM_LIMBS_255BIT];
+        x_2 = new int[NUM_LIMBS_255BIT];
+        x_3 = new int[NUM_LIMBS_255BIT];
+        z_2 = new int[NUM_LIMBS_255BIT];
+        z_3 = new int[NUM_LIMBS_255BIT];
+        A = new int[NUM_LIMBS_255BIT];
+        B = new int[NUM_LIMBS_255BIT];
+        C = new int[NUM_LIMBS_255BIT];
+        D = new int[NUM_LIMBS_255BIT];
+        E = new int[NUM_LIMBS_255BIT];
+        AA = new int[NUM_LIMBS_255BIT];
+        BB = new int[NUM_LIMBS_255BIT];
+        DA = new int[NUM_LIMBS_255BIT];
+        CB = new int[NUM_LIMBS_255BIT];
+        t1 = new long[NUM_LIMBS_510BIT];
+        t2 = new int[NUM_LIMBS_510BIT];
     }
 
 
@@ -102,8 +101,8 @@ public final class Curve25519 {
         Arrays.fill(BB, 0);
         Arrays.fill(DA, 0);
         Arrays.fill(CB, 0);
-        Arrays.fill(t1,  0L);
-        Arrays.fill(t2,  0);
+        Arrays.fill(t1, 0L);
+        Arrays.fill(t2, 0);
     }
 
     /**
@@ -112,8 +111,7 @@ public final class Curve25519 {
      *
      * @param x The number to reduce, and the result.
      */
-    private void reduceQuick(int[] x)
-    {
+    private void reduceQuick(int[] x) {
         int index, carry;
 
         // Perform a trial subtraction of (2^255 - 19) from "x" which is
@@ -142,12 +140,11 @@ public final class Curve25519 {
      * Reduce a number modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The value to be reduced.  This array will be
-     * modified during the reduction.
-     * @param size The number of limbs in the high order half of x.
+     * @param x      The value to be reduced.  This array will be
+     *               modified during the reduction.
+     * @param size   The number of limbs in the high order half of x.
      */
-    private void reduce(int[] result, int[] x, int size)
-    {
+    private void reduce(int[] result, int[] x, int size) {
         int index, limb, carry;
 
         // Calculate (x mod 2^255) + ((x / 2^255) * 19) which will
@@ -198,11 +195,10 @@ public final class Curve25519 {
      * Multiplies two numbers modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The first number to multiply.
-     * @param y The second number to multiply.
+     * @param x      The first number to multiply.
+     * @param y      The second number to multiply.
      */
-    private void mul(int[] result, int[] x, int[] y)
-    {
+    private void mul(int[] result, int[] x, int[] y) {
         int i, j;
 
         // Multiply the two numbers to create the intermediate result.
@@ -220,10 +216,10 @@ public final class Curve25519 {
 
         // Propagate carries and convert back into 26-bit words.
         v = t1[0];
-        t2[0] = ((int)v) & 0x03FFFFFF;
+        t2[0] = ((int) v) & 0x03FFFFFF;
         for (i = 1; i < NUM_LIMBS_510BIT; ++i) {
             v = (v >> 26) + t1[i];
-            t2[i] = ((int)v) & 0x03FFFFFF;
+            t2[i] = ((int) v) & 0x03FFFFFF;
         }
 
         // Reduce the result modulo 2^255 - 19.
@@ -234,10 +230,9 @@ public final class Curve25519 {
      * Squares a number modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The number to square.
+     * @param x      The number to square.
      */
-    private void square(int[] result, int[] x)
-    {
+    private void square(int[] result, int[] x) {
         mul(result, x, x);
     }
 
@@ -245,19 +240,18 @@ public final class Curve25519 {
      * Multiplies a number by the a24 constant, modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The number to multiply by a24.
+     * @param x      The number to multiply by a24.
      */
-    private void mulA24(int[] result, int[] x)
-    {
+    private void mulA24(int[] result, int[] x) {
         long a24 = 121665;
         long carry = 0;
         int index;
         for (index = 0; index < NUM_LIMBS_255BIT; ++index) {
             carry += a24 * x[index];
-            t2[index] = ((int)carry) & 0x03FFFFFF;
+            t2[index] = ((int) carry) & 0x03FFFFFF;
             carry >>= 26;
         }
-        t2[NUM_LIMBS_255BIT] = ((int)carry) & 0x03FFFFFF;
+        t2[NUM_LIMBS_255BIT] = ((int) carry) & 0x03FFFFFF;
         reduce(result, t2, 1);
     }
 
@@ -265,11 +259,10 @@ public final class Curve25519 {
      * Adds two numbers modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The first number to add.
-     * @param y The second number to add.
+     * @param x      The first number to add.
+     * @param y      The second number to add.
      */
-    private void add(int[] result, int[] x, int[] y)
-    {
+    private void add(int[] result, int[] x, int[] y) {
         int index, carry;
         carry = x[0] + y[0];
         result[0] = carry & 0x03FFFFFF;
@@ -284,11 +277,10 @@ public final class Curve25519 {
      * Subtracts two numbers modulo 2^255 - 19.
      *
      * @param result The result.
-     * @param x The first number to subtract.
-     * @param y The second number to subtract.
+     * @param x      The first number to subtract.
+     * @param y      The second number to subtract.
      */
-    private void sub(int[] result, int[] x, int[] y)
-    {
+    private void sub(int[] result, int[] x, int[] y) {
         int index, borrow;
 
         // Subtract y from x to generate the intermediate result.
@@ -316,11 +308,10 @@ public final class Curve25519 {
      * Conditional swap of two values.
      *
      * @param select Set to 1 to swap, 0 to leave as-is.
-     * @param x The first value.
-     * @param y The second value.
+     * @param x      The first value.
+     * @param y      The second value.
      */
-    private static void cswap(int select, int[] x, int[] y)
-    {
+    private static void cswap(int select, int[] x, int[] y) {
         int dummy;
         select = -select;
         for (int index = 0; index < NUM_LIMBS_255BIT; ++index) {
@@ -334,10 +325,9 @@ public final class Curve25519 {
      * Raise x to the power of (2^250 - 1).
      *
      * @param result The result.  Must not overlap with x.
-     * @param x The argument.
+     * @param x      The argument.
      */
-    private void pow250(int[] result, int[] x)
-    {
+    private void pow250(int[] result, int[] x) {
         int i, j;
 
         // The big-endian hexadecimal expansion of (2^250 - 1) is:
@@ -375,10 +365,9 @@ public final class Curve25519 {
      * Computes the reciprocal of a number modulo 2^255 - 19.
      *
      * @param result The result.  Must not overlap with x.
-     * @param x The argument.
+     * @param x      The argument.
      */
-    private void recip(int[] result, int[] x)
-    {
+    private void recip(int[] result, int[] x) {
         // The reciprocal is the same as x ^ (p - 2) where p = 2^255 - 19.
         // The big-endian hexadecimal expansion of (p - 2) is:
         // 7FFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFEB
@@ -401,8 +390,7 @@ public final class Curve25519 {
      *
      * @param s The 32-byte secret key.
      */
-    private void evalCurve(byte[] s)
-    {
+    private void evalCurve(byte[] s) {
         int sposn = 31;
         int sbit = 6;
         int svalue = s[sposn] | 0x40;
@@ -411,7 +399,7 @@ public final class Curve25519 {
 
         // Iterate over all 255 bits of "s" from the highest to the lowest.
         // We ignore the high bit of the 256-bit representation of "s".
-        for (;;) {
+        for (; ; ) {
             // Conditional swaps on entry to this bit but only if we
             // didn't swap on the previous bit.
             select = (svalue >> sbit) & 0x01;
@@ -464,14 +452,13 @@ public final class Curve25519 {
     /**
      * Evaluates the Curve25519 curve.
      *
-     * @param result Buffer to place the result of the evaluation into.
-     * @param offset Offset into the result buffer.
+     * @param result     Buffer to place the result of the evaluation into.
+     * @param offset     Offset into the result buffer.
      * @param privateKey The private key to use in the evaluation.
-     * @param publicKey The public key to use in the evaluation, or null
-     * if the base point of the curve should be used.
+     * @param publicKey  The public key to use in the evaluation, or null
+     *                   if the base point of the curve should be used.
      */
-    public static void eval(byte[] result, int offset, byte[] privateKey, byte[] publicKey)
-    {
+    public static void eval(byte[] result, int offset, byte[] privateKey, byte[] publicKey) {
         Curve25519 state = new Curve25519();
         try {
             // Unpack the public key value.  If null, use 9 as the base point.
@@ -501,11 +488,11 @@ public final class Curve25519 {
             }
 
             // Initialize the other temporary variables.
-            Arrays.fill(state.x_2, 0);			// x_2 = 1
+            Arrays.fill(state.x_2, 0);            // x_2 = 1
             state.x_2[0] = 1;
-            Arrays.fill(state.z_2, 0);			// z_2 = 0
+            Arrays.fill(state.z_2, 0);            // z_2 = 0
             System.arraycopy(state.x_1, 0, state.x_3, 0, state.x_1.length);  // x_3 = x_1
-            Arrays.fill(state.z_3, 0);			// z_3 = 1
+            Arrays.fill(state.z_3, 0);            // z_3 = 1
             state.z_3[0] = 1;
 
             // Evaluate the curve for every bit of the private key.
@@ -520,9 +507,9 @@ public final class Curve25519 {
                 int bit = (index * 8) % 26;
                 int word = (index * 8) / 26;
                 if (bit <= (26 - 8))
-                    result[offset + index] = (byte)(state.x_2[word] >> bit);
+                    result[offset + index] = (byte) (state.x_2[word] >> bit);
                 else
-                    result[offset + index] = (byte)((state.x_2[word] >> bit) | (state.x_2[word + 1] << (26 - bit)));
+                    result[offset + index] = (byte) ((state.x_2[word] >> bit) | (state.x_2[word + 1] << (26 - bit)));
             }
         } finally {
             // Clean up all temporary state before we exit.

eclair-core/src/test/scala/fr/acinq/eclair/FeaturesSpec.scala

@@ -2,7 +2,7 @@ package fr.acinq.eclair
 
 import java.nio.ByteOrder
 
-import fr.acinq.bitcoin.{BinaryData, Protocol}
+import fr.acinq.bitcoin.Protocol
 import fr.acinq.eclair.Features._
 import org.junit.runner.RunWith
 import org.scalatest.FunSuite

eclair-core/src/test/scala/fr/acinq/eclair/Pipe.scala

@@ -2,7 +2,6 @@ package fr.acinq.eclair
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Stash}
 import fr.acinq.eclair.channel.Commitments.msg2String
-import fr.acinq.eclair.channel.{INPUT_DISCONNECTED, INPUT_RECONNECTED}
 import fr.acinq.eclair.wire.LightningMessage
 
 /**

eclair-core/src/test/scala/fr/acinq/eclair/TestBitcoinClient.scala

@@ -3,7 +3,7 @@ package fr.acinq.eclair
 import akka.actor.ActorSystem
 import fr.acinq.bitcoin.{Block, Transaction}
 import fr.acinq.eclair.blockchain._
-import fr.acinq.eclair.blockchain.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}

eclair-core/src/test/scala/fr/acinq/eclair/TestConstants.scala

@@ -5,7 +5,8 @@ import java.sql.DriverManager
 
 import fr.acinq.bitcoin.Crypto.PrivateKey
 import fr.acinq.bitcoin.{BinaryData, Block, DeterministicWallet, Script}
-import fr.acinq.eclair.db.sqlite.{SqliteChannelsDb, SqliteNetworkDb, SqlitePeersDb}
+import fr.acinq.eclair.NodeParams.BITCOIND
+import fr.acinq.eclair.db.sqlite.{SqliteChannelsDb, SqliteNetworkDb, SqlitePeersDb, SqlitePreimagesDb}
 import fr.acinq.eclair.io.Peer
 
 import scala.concurrent.duration._
@@ -22,7 +23,9 @@ object TestConstants {
     val seed = BinaryData("01" * 32)
     val master = DeterministicWallet.generate(seed)
     val extendedPrivateKey = DeterministicWallet.derivePrivateKey(master, DeterministicWallet.hardened(46) :: DeterministicWallet.hardened(0) :: Nil)
+
     def sqlite = DriverManager.getConnection("jdbc:sqlite::memory:")
+
     def nodeParams = NodeParams(
       extendedPrivateKey = extendedPrivateKey,
       privateKey = extendedPrivateKey.privateKey,
@@ -46,6 +49,7 @@ object TestConstants {
       channelsDb = new SqliteChannelsDb(sqlite),
       peersDb = new SqlitePeersDb(sqlite),
       networkDb = new SqliteNetworkDb(sqlite),
+      preimagesDb = new SqlitePreimagesDb(sqlite),
       routerBroadcastInterval = 60 seconds,
       routerValidateInterval = 2 seconds,
       pingInterval = 30 seconds,
@@ -55,8 +59,10 @@ object TestConstants {
       chainHash = Block.RegtestGenesisBlock.hash,
       channelFlags = 1,
       channelExcludeDuration = 5 seconds,
-      spv = false)
+      watcherType = BITCOIND)
+
     def id = nodeParams.privateKey.publicKey
+
     def channelParams = Peer.makeChannelParams(
       nodeParams = nodeParams,
       defaultFinalScriptPubKey = Script.write(Script.pay2wpkh(PrivateKey(Array.fill[Byte](32)(4), compressed = true).publicKey)),
@@ -70,7 +76,9 @@ object TestConstants {
     val seed = BinaryData("02" * 32)
     val master = DeterministicWallet.generate(seed)
     val extendedPrivateKey = DeterministicWallet.derivePrivateKey(master, DeterministicWallet.hardened(46) :: DeterministicWallet.hardened(0) :: Nil)
+
     def sqlite = DriverManager.getConnection("jdbc:sqlite::memory:")
+
     def nodeParams = NodeParams(
       extendedPrivateKey = extendedPrivateKey,
       privateKey = extendedPrivateKey.privateKey,
@@ -94,6 +102,7 @@ object TestConstants {
       channelsDb = new SqliteChannelsDb(sqlite),
       peersDb = new SqlitePeersDb(sqlite),
       networkDb = new SqliteNetworkDb(sqlite),
+      preimagesDb = new SqlitePreimagesDb(sqlite),
       routerBroadcastInterval = 60 seconds,
       routerValidateInterval = 2 seconds,
       pingInterval = 30 seconds,
@@ -103,8 +112,10 @@ object TestConstants {
       chainHash = Block.RegtestGenesisBlock.hash,
       channelFlags = 1,
       channelExcludeDuration = 5 seconds,
-      spv = false)
+      watcherType = BITCOIND)
+
     def id = nodeParams.privateKey.publicKey
+
     def channelParams = Peer.makeChannelParams(
       nodeParams = nodeParams,
       defaultFinalScriptPubKey = Script.write(Script.pay2wpkh(PrivateKey(Array.fill[Byte](32)(5), compressed = true).publicKey)),

eclair-core/src/test/scala/fr/acinq/eclair/TestkitBaseClass.scala

@@ -2,6 +2,7 @@ package fr.acinq.eclair
 
 import akka.actor.{ActorNotFound, ActorSystem, PoisonPill}
 import akka.testkit.TestKit
+import fr.acinq.eclair.blockchain.fee.FeeratesPerKw
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, fixture}
 
 import scala.concurrent.Await
@@ -14,7 +15,7 @@ abstract class TestkitBaseClass extends TestKit(ActorSystem("test")) with fixtur
 
   override def beforeAll {
     Globals.blockCount.set(400000)
-    Globals.feeratePerKw.set(TestConstants.feeratePerKw)
+    Globals.feeratesPerKw.set(FeeratesPerKw.single(TestConstants.feeratePerKw))
   }
 
   override def afterEach() {
@@ -27,7 +28,7 @@ abstract class TestkitBaseClass extends TestKit(ActorSystem("test")) with fixtur
 
   override def afterAll {
     TestKit.shutdownActorSystem(system)
-    Globals.feeratePerKw.set(0)
+    Globals.feeratesPerKw.set(FeeratesPerKw.single(0))
   }
 
 }

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/ExtendedBitcoinClientSpec.scala

@@ -2,11 +2,11 @@ package fr.acinq.eclair.blockchain
 
 import akka.actor.ActorSystem
 import com.typesafe.config.ConfigFactory
-import fr.acinq.eclair.blockchain.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, ExtendedBitcoinClient}
 import org.scalatest.FunSuite
 
-import scala.concurrent.{Await, ExecutionContext}
 import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext}
 
 // this test is not run automatically
 class ExtendedBitcoinClientSpec extends FunSuite {

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/TestWallet.scala

@@ -1,9 +1,8 @@
 package fr.acinq.eclair.blockchain
 
 import fr.acinq.bitcoin.{BinaryData, Crypto, OP_PUSHDATA, OutPoint, Satoshi, Script, Transaction, TxIn, TxOut}
-import fr.acinq.eclair.blockchain.wallet.{EclairWallet, MakeFundingTxResponse}
 
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.Future
 import scala.util.Try
 
 /**
@@ -19,6 +18,8 @@ class TestWallet extends EclairWallet {
     Future.successful(TestWallet.makeDummyFundingTx(pubkeyScript, amount, feeRatePerKw))
 
   override def commit(tx: Transaction): Future[Boolean] = Future.successful(true)
+
+  override def rollback(tx: Transaction): Future[Boolean] = Future.successful(true)
 }
 
 object TestWallet {

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoinj/BitcoinjSpec.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.blockchain
+package fr.acinq.eclair.blockchain.bitcoinj
 
 import java.io.File
 import java.net.InetSocketAddress
@@ -9,16 +9,13 @@ import akka.actor.{Actor, ActorRef, ActorSystem, Props}
 import akka.pattern.pipe
 import akka.testkit.{TestKit, TestProbe}
 import fr.acinq.bitcoin.{Satoshi, Script}
-import fr.acinq.eclair.blockchain.rpc.BitcoinJsonRPCClient
-import fr.acinq.eclair.blockchain.spv.BitcoinjKit
-import fr.acinq.eclair.blockchain.wallet.BitcoinjWallet
+import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinJsonRPCClient
+import fr.acinq.eclair.blockchain.{PublishAsap, WatchConfirmed, WatchEventConfirmed, WatchSpent}
 import fr.acinq.eclair.channel.{BITCOIN_FUNDING_DEPTHOK, BITCOIN_FUNDING_SPENT}
 import fr.acinq.eclair.randomKey
 import fr.acinq.eclair.transactions.Scripts
 import grizzled.slf4j.Logging
-import org.bitcoinj.core.{Coin, Transaction}
 import org.bitcoinj.script.{Script => BitcoinjScript}
-import org.bitcoinj.wallet.{SendRequest, Wallet}
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.JValue
 import org.junit.runner.RunWith
@@ -84,7 +81,7 @@ class BitcoinjSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with B
     }, max = 30 seconds, interval = 500 millis)
     logger.info(s"generating initial blocks...")
     sender.send(bitcoincli, BitcoinReq("generate", 500))
-    sender.expectMsgType[JValue](10 seconds)
+    sender.expectMsgType[JValue](30 seconds)
   }
 
   ignore("bitcoinj wallet commit") {
@@ -143,7 +140,7 @@ class BitcoinjSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with B
     bitcoinjKit.awaitRunning()
 
     val sender = TestProbe()
-    val watcher = system.actorOf(Props(new SpvWatcher(bitcoinjKit)), name = "spv-watcher")
+    val watcher = system.actorOf(Props(new BitcoinjWatcher(bitcoinjKit)), name = "bitcoinj-watcher")
     val wallet = new BitcoinjWallet(Future.successful(bitcoinjKit.wallet()))
 
     val address = Await.result(wallet.getFinalAddress, 10 seconds)
@@ -159,10 +156,9 @@ class BitcoinjSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with B
     val listener = TestProbe()
     val fundingPubkeyScript = Script.write(Script.pay2wsh(Scripts.multiSig2of2(randomKey.publicKey, randomKey.publicKey)))
     val result = Await.result(wallet.makeFundingTx(fundingPubkeyScript, Satoshi(10000L), 20000), 10 seconds)
-    watcher ! Hint(new BitcoinjScript(fundingPubkeyScript))
     assert(Await.result(wallet.commit(result.fundingTx), 10 seconds))
-    watcher ! WatchSpent(listener.ref, result.fundingTx.txid, result.fundingTxOutputIndex, BITCOIN_FUNDING_SPENT)
-    watcher ! WatchConfirmed(listener.ref, result.fundingTx.txid, 3, BITCOIN_FUNDING_DEPTHOK)
+    watcher ! WatchSpent(listener.ref, result.fundingTx, result.fundingTxOutputIndex, BITCOIN_FUNDING_SPENT)
+    watcher ! WatchConfirmed(listener.ref, result.fundingTx, 3, BITCOIN_FUNDING_DEPTHOK)
     watcher ! PublishAsap(result.fundingTx)
 
     logger.info(s"waiting for confirmation of ${result.fundingTx.txid}")
@@ -177,7 +173,7 @@ class BitcoinjSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with B
     bitcoinjKit.awaitRunning()
 
     val sender = TestProbe()
-    val watcher = system.actorOf(Props(new SpvWatcher(bitcoinjKit)), name = "spv-watcher")
+    val watcher = system.actorOf(Props(new BitcoinjWatcher(bitcoinjKit)), name = "bitcoinj-watcher")
     val wallet = new BitcoinjWallet(Future.successful(bitcoinjKit.wallet()))
 
     val address = Await.result(wallet.getFinalAddress, 10 seconds)
@@ -193,10 +189,9 @@ class BitcoinjSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with B
           val listener = TestProbe()
           val fundingPubkeyScript = Script.write(Script.pay2wsh(Scripts.multiSig2of2(randomKey.publicKey, randomKey.publicKey)))
           val result = Await.result(wallet.makeFundingTx(fundingPubkeyScript, Satoshi(10000L), 20000), 10 seconds)
-          watcher ! Hint(new BitcoinjScript(fundingPubkeyScript))
           assert(Await.result(wallet.commit(result.fundingTx), 10 seconds))
-          watcher ! WatchSpent(listener.ref, result.fundingTx.txid, result.fundingTxOutputIndex, BITCOIN_FUNDING_SPENT)
-          watcher ! WatchConfirmed(listener.ref, result.fundingTx.txid, 3, BITCOIN_FUNDING_DEPTHOK)
+          watcher ! WatchSpent(listener.ref, result.fundingTx, result.fundingTxOutputIndex, BITCOIN_FUNDING_SPENT)
+          watcher ! WatchConfirmed(listener.ref, result.fundingTx, 3, BITCOIN_FUNDING_DEPTHOK)
           watcher ! PublishAsap(result.fundingTx)
           (result.fundingTx.txid, listener)
       }

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumClientSpec.scala

@@ -0,0 +1,79 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import akka.actor.{ActorRef, ActorSystem, Props}
+import akka.testkit.{TestKit, TestProbe}
+import fr.acinq.bitcoin.{BinaryData, Crypto, Transaction}
+import grizzled.slf4j.Logging
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import scala.concurrent.duration._
+
+@RunWith(classOf[JUnitRunner])
+class ElectrumClientSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with Logging with BeforeAndAfterAll {
+
+  import ElectrumClient._
+
+  var client: ActorRef = _
+  val probe = TestProbe()
+  val referenceTx = Transaction.read("0200000003947e307df3ab452d23f02b5a65f4ada1804ee733e168e6197b0bd6cc79932b6c010000006a473044022069346ec6526454a481690a3664609f9e8032c34553015cfa2e9b25ebb420a33002206998f21a2aa771ad92a0c1083f4181a3acdb0d42ca51d01be1309da2ffb9cecf012102b4568cc6ee751f6d39f4a908b1fcffdb878f5f784a26a48c0acb0acff9d88e3bfeffffff966d9d969cd5f95bfd53003a35fcc1a50f4fb51f211596e6472583fdc5d38470000000006b4830450221009c9757515009c5709b5b678d678185202b817ef9a69ffb954144615ab11762210220732216384da4bf79340e9c46d0effba6ba92982cca998adfc3f354cec7715f800121035f7c3e077108035026f4ebd5d6ca696ef088d4f34d45d94eab4c41202ec74f9bfefffffff8d5062f5b04455c6cfa7e3f250e5a4fb44308ba2b86baf77f9ad0d782f57071010000006a47304402207f9f7dd91fe537a26d5554105977e3949a5c8c4ef53a6a3bff6da2d36eff928f02202b9427bef487a1825fd0c3c6851d17d5f19e6d73dfee22bf06db591929a2044d012102b4568cc6ee751f6d39f4a908b1fcffdb878f5f784a26a48c0acb0acff9d88e3bfeffffff02809698000000000017a914c82753548fdf4be1c3c7b14872c90b5198e67eaa876e642500000000001976a914e2365ec29471b3e271388b22eadf0e7f54d307a788ac6f771200")
+  val scriptHash: BinaryData = Crypto.sha256(referenceTx.txOut(0).publicKeyScript).reverse
+
+  override protected def beforeAll(): Unit = {
+    val stream = classOf[ElectrumClientSpec].getResourceAsStream("/electrum/servers_testnet.json")
+    val addresses = ElectrumClient.readServerAddresses(stream)
+    stream.close()
+    client = system.actorOf(Props(new ElectrumClient(addresses)), "electrum-client")
+  }
+
+  override protected def afterAll(): Unit = {
+    TestKit.shutdownActorSystem(system)
+  }
+
+  test("connect to an electrumx testnet server") {
+    probe.send(client, AddStatusListener(probe.ref))
+    probe.expectMsg(5 seconds, ElectrumReady)
+  }
+
+  test("get transaction") {
+    probe.send(client, GetTransaction("c5efb5cbd35a44ba956b18100be0a91c9c33af4c7f31be20e33741d95f04e202"))
+    val GetTransactionResponse(tx) = probe.expectMsgType[GetTransactionResponse]
+    assert(tx.txid == BinaryData("c5efb5cbd35a44ba956b18100be0a91c9c33af4c7f31be20e33741d95f04e202"))
+  }
+
+  test("get merkle tree") {
+    probe.send(client, GetMerkle("c5efb5cbd35a44ba956b18100be0a91c9c33af4c7f31be20e33741d95f04e202", 1210223L))
+    val response = probe.expectMsgType[GetMerkleResponse]
+    assert(response.txid == BinaryData("c5efb5cbd35a44ba956b18100be0a91c9c33af4c7f31be20e33741d95f04e202"))
+    assert(response.block_height == 1210223L)
+    assert(response.pos == 28)
+    assert(response.root == BinaryData("fb0234a21e96913682bc4108bcf72b67fb5d2dd680875b7e4671c03ccf523a20"))
+  }
+
+  test("header subscription") {
+    val probe1 = TestProbe()
+    probe1.send(client, HeaderSubscription(probe1.ref))
+    val HeaderSubscriptionResponse(header) = probe1.expectMsgType[HeaderSubscriptionResponse]
+    logger.info(s"received header for block ${header.block_hash}")
+  }
+
+  test("scripthash subscription") {
+    val probe1 = TestProbe()
+    probe1.send(client, ScriptHashSubscription(scriptHash, probe1.ref))
+    val ScriptHashSubscriptionResponse(scriptHash1, status) = probe1.expectMsgType[ScriptHashSubscriptionResponse]
+    assert(status != "")
+  }
+
+  test("get scripthash history") {
+    probe.send(client, GetScriptHashHistory(scriptHash))
+    val GetScriptHashHistoryResponse(scriptHash1, history) = probe.expectMsgType[GetScriptHashHistoryResponse]
+    assert(history.contains((TransactionHistoryItem(1210224, "3903726806aa044fe59f40e42eed71bded068b43aaa9e2d716e38b7825412de0"))))
+  }
+
+  test("list script unspents") {
+    probe.send(client, ScriptHashListUnspent(scriptHash))
+    val ScriptHashListUnspentResponse(scriptHash1, unspents) = probe.expectMsgType[ScriptHashListUnspentResponse]
+    assert(unspents.contains(UnspentItem("3903726806aa044fe59f40e42eed71bded068b43aaa9e2d716e38b7825412de0", 0, 10000000L, 1210224L)))
+  }
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletBaseSpec.scala

@@ -0,0 +1,123 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import java.io.File
+import java.net.InetSocketAddress
+import java.nio.file.Files
+import java.util.UUID
+
+import akka.actor.{Actor, ActorRef, ActorSystem, Props}
+import akka.pattern.pipe
+import akka.testkit.{TestKit, TestProbe}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinJsonRPCClient
+import grizzled.slf4j.Logging
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST.{JInt, JValue}
+import org.json4s.jackson.JsonMethods
+import org.junit.Ignore
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.sys.process._
+import scala.util.{Success, Try}
+
+@Ignore
+class IntegrationSpec extends TestKit(ActorSystem("test")) with FunSuiteLike with BeforeAndAfterAll with Logging {
+  implicit val formats = DefaultFormats
+
+  require(System.getProperty("buildDirectory") != null, "please define system property buildDirectory")
+  require(System.getProperty("electrumxPath") != null, "please define system property electrumxPath")
+
+  val INTEGRATION_TMP_DIR = s"${System.getProperty("buildDirectory")}/integration-${UUID.randomUUID().toString}"
+  logger.info(s"using tmp dir: $INTEGRATION_TMP_DIR")
+
+  val PATH_BITCOIND = new File(System.getProperty("buildDirectory"), "bitcoin-0.15.0/bin/bitcoind")
+  val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin")
+  val PATH_ELECTRUMX_DBDIR = new File(INTEGRATION_TMP_DIR, "electrumx-db")
+  val PATH_ELECTRUMX = new File(System.getProperty("electrumxPath"))
+
+  var bitcoind: Process = _
+  var bitcoinrpcclient: BitcoinJsonRPCClient = _
+  var bitcoincli: ActorRef = _
+
+  var elecxtrumx: Process = _
+  var electrumClient: ActorRef = _
+
+  case class BitcoinReq(method: String, params: Seq[Any] = Nil)
+
+  override protected def beforeAll(): Unit = {
+    Files.createDirectories(PATH_BITCOIND_DATADIR.toPath)
+    Files.copy(classOf[IntegrationSpec].getResourceAsStream("/integration/bitcoin.conf"), new File(PATH_BITCOIND_DATADIR.toString, "bitcoin.conf").toPath)
+
+    bitcoinrpcclient = new BitcoinJsonRPCClient(user = "foo", password = "bar", host = "localhost", port = 28332)
+    bitcoincli = system.actorOf(Props(new Actor {
+      override def receive: Receive = {
+        case BitcoinReq(method, Nil) =>
+          bitcoinrpcclient.invoke(method) pipeTo sender
+        case BitcoinReq(method, params) =>
+          bitcoinrpcclient.invoke(method, params: _*) pipeTo sender
+      }
+    }))
+    Files.createDirectories(PATH_ELECTRUMX_DBDIR.toPath)
+    startBitcoind
+    logger.info(s"generating initial blocks...")
+    val sender = TestProbe()
+    sender.send(bitcoincli, BitcoinReq("generate", 500 :: Nil))
+    sender.expectMsgType[JValue](10 seconds)
+
+    startElectrum
+    electrumClient = system.actorOf(Props(new ElectrumClient(Seq(new InetSocketAddress("localhost", 51001)))))
+    sender.send(electrumClient, ElectrumClient.AddStatusListener(sender.ref))
+    sender.expectMsg(3 seconds, ElectrumClient.ElectrumReady)
+  }
+
+  override protected def afterAll(): Unit = {
+    logger.info(s"stopping bitcoind")
+    stopBitcoind
+    bitcoind.destroy()
+    logger.info(s"stopping electrumx")
+    elecxtrumx.destroy()
+  }
+
+  def startBitcoind: Unit = {
+    bitcoind = s"$PATH_BITCOIND -datadir=$PATH_BITCOIND_DATADIR".run()
+    val sender = TestProbe()
+    logger.info(s"waiting for bitcoind to initialize...")
+    awaitCond({
+      sender.send(bitcoincli, BitcoinReq("getnetworkinfo"))
+      sender.receiveOne(5 second).isInstanceOf[JValue]
+    }, max = 30 seconds, interval = 500 millis)
+    logger.info(s"bitcoind is ready")
+  }
+
+  def stopBitcoind: Unit = {
+    // gracefully stopping bitcoin will make it store its state cleanly to disk, which is good for later debugging
+    val sender = TestProbe()
+    sender.send(bitcoincli, BitcoinReq("stop"))
+    bitcoind.exitValue()
+  }
+
+  def restartBitcoind: Unit = {
+    stopBitcoind
+    startBitcoind
+  }
+
+  def startElectrum: Unit = {
+    elecxtrumx = Process(s"$PATH_ELECTRUMX/electrumx_server.py",
+      None,
+      "DB_DIRECTORY" -> PATH_ELECTRUMX_DBDIR.getAbsolutePath,
+      "DAEMON_URL" -> "foo:bar@localhost:28332",
+      "COIN" -> "BitcoinSegwit",
+      "NET" -> "regtest",
+      "TCP_PORT" -> "51001").run()
+
+    logger.info(s"waiting for electrumx to initialize...")
+    awaitCond({
+      val result = s"$PATH_ELECTRUMX/electrumx_rpc.py getinfo".!!
+      Try(JsonMethods.parse(result) \ "daemon_height") match {
+        case Success(JInt(value)) if value.intValue() == 500 => true
+        case _ => false
+      }
+    }, max = 30 seconds, interval = 500 millis)
+  }
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletBasicSpec.scala

@@ -0,0 +1,98 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import fr.acinq.bitcoin.Crypto.PrivateKey
+import fr.acinq.bitcoin.DeterministicWallet.derivePrivateKey
+import fr.acinq.bitcoin._
+import org.junit.runner.RunWith
+import org.scalatest.FunSuite
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class ElectrumWalletBasicSpec extends FunSuite {
+
+  import ElectrumWallet._
+
+  val swipeRange = 10
+  val dustLimit = 546 satoshi
+  val feeRatePerKw = 20000
+  val minimumFee = Satoshi(2000)
+
+  val master = DeterministicWallet.generate(BinaryData("01" * 32))
+  val accountMaster = accountKey(master)
+  val accountIndex = 0
+
+  val changeMaster = changeKey(master)
+  val changeIndex = 0
+
+  val firstAccountKeys = (0 until 10).map(i => derivePrivateKey(accountMaster, i)).toVector
+  val firstChangeKeys = (0 until 10).map(i => derivePrivateKey(changeMaster, i)).toVector
+
+  val params = ElectrumWallet.WalletParameters(Block.RegtestGenesisBlock.hash)
+
+  val state = Data(params, ElectrumClient.Header.RegtestGenesisHeader, firstAccountKeys, firstChangeKeys)
+  val unspents = Map(
+    computeScriptHashFromPublicKey(state.accountKeys(0).publicKey) -> Set(ElectrumClient.UnspentItem("01" * 32, 0, 1 * Satoshi(Coin).toLong, 100)),
+    computeScriptHashFromPublicKey(state.accountKeys(1).publicKey) -> Set(ElectrumClient.UnspentItem("02" * 32, 0, 2 * Satoshi(Coin).toLong, 100)),
+    computeScriptHashFromPublicKey(state.accountKeys(2).publicKey) -> Set(ElectrumClient.UnspentItem("03" * 32, 0, 3 * Satoshi(Coin).toLong, 100))
+  )
+
+  test("compute addresses") {
+    val priv = PrivateKey.fromBase58("cRumXueoZHjhGXrZWeFoEBkeDHu2m8dW5qtFBCqSAt4LDR2Hnd8Q", Base58.Prefix.SecretKeyTestnet)
+    assert(Base58Check.encode(Base58.Prefix.PubkeyAddressTestnet, priv.publicKey.hash160) == "ms93boMGZZjvjciujPJgDAqeR86EKBf9MC")
+    assert(segwitAddress(priv) == "2MscvqgGXMTYJNAY3owdUtgWJaxPUjH38Cx")
+  }
+
+  test("implement BIP49") {
+    val mnemonics = "pizza afraid guess romance pair steel record jazz rubber prison angle hen heart engage kiss visual helmet twelve lady found between wave rapid twist".split(" ")
+    val seed = MnemonicCode.toSeed(mnemonics, "")
+    val master = DeterministicWallet.generate(seed)
+
+    val accountMaster = accountKey(master)
+    val firstKey = derivePrivateKey(accountMaster, 0)
+    assert(segwitAddress(firstKey) === "2MxJejujQJRRJdbfTKNQQ94YCnxJwRaE7yo")
+  }
+
+  ignore("complete transactions (enough funds)") {
+    val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
+
+    val pub = PrivateKey(BinaryData("01" * 32), compressed = true).publicKey
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(pub)) :: Nil, lockTime = 0)
+    val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
+
+    val state3 = state2.cancelTransaction(tx1)
+    assert(state3 == state1)
+
+    val state4 = state2.commitTransaction(tx1)
+    assert(state4.utxos.size + tx1.txIn.size == state1.utxos.size)
+  }
+
+  test("complete transactions (insufficient funds)") {
+    val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(6 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
+    val e = intercept[IllegalArgumentException] {
+      val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
+    }
+  }
+
+  ignore("find what a tx spends from us") {
+    val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
+    val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
+
+    val pubkeys = tx1.txIn.map(extractPubKeySpentFrom).flatten
+    val utxos1 = state2.utxos.filter(utxo => pubkeys.contains(utxo.key.publicKey))
+    val utxos2 = state2.utxos.filter(utxo => tx1.txIn.map(_.outPoint).contains(utxo.outPoint))
+    println(pubkeys)
+  }
+
+  ignore("find what a tx sends to us") {
+    val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
+    val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
+
+    val pubSpent = tx1.txIn.map(extractPubKeySpentFrom).flatten
+    val utxos1 = state2.utxos.filter(utxo => pubSpent.contains(utxo.key.publicKey))
+    val utxos2 = state2.utxos.filter(utxo => tx1.txIn.map(_.outPoint).contains(utxo.outPoint))
+    println(pubSpent)
+  }
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletSpec.scala

@@ -0,0 +1,300 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import akka.actor.{ActorRef, Props}
+import akka.testkit.TestProbe
+import fr.acinq.bitcoin._
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient.{AddStatusListener, BroadcastTransaction, BroadcastTransactionResponse}
+import org.json4s.JsonAST._
+
+import scala.concurrent.duration._
+import scala.sys.process._
+
+class ElectrumWalletSpec extends IntegrationSpec {
+
+  import ElectrumWallet._
+
+  val entropy = BinaryData("01" * 32)
+  val mnemonics = MnemonicCode.toMnemonics(entropy)
+  val seed = MnemonicCode.toSeed(mnemonics, "")
+  logger.info(s"mnemonic codes for our wallet: $mnemonics")
+  val master = DeterministicWallet.generate(seed)
+  var wallet: ActorRef = _
+
+  test("wait until wallet is ready") {
+    wallet = system.actorOf(Props(new ElectrumWallet(mnemonics, electrumClient, WalletParameters(Block.RegtestGenesisBlock.hash, minimumFee = Satoshi(5000)))), "wallet")
+    val probe = TestProbe()
+    awaitCond({
+      probe.send(wallet, GetData)
+      val GetDataResponse(state) = probe.expectMsgType[GetDataResponse]
+      state.status.size == state.accountKeys.size + state.changeKeys.size
+    }, max = 30 seconds, interval = 1 second)
+    logger.info(s"wallet is ready")
+  }
+
+  ignore("receive funds") {
+    val probe = TestProbe()
+
+    probe.send(wallet, GetCurrentReceiveAddress)
+    val GetCurrentReceiveAddressResponse(address) = probe.expectMsgType[GetCurrentReceiveAddressResponse]
+
+    logger.info(s"sending 1 btc to $address")
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address :: 1.0 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+      unconfirmed == Satoshi(100000000L)
+    }, max = 30 seconds, interval = 1 second)
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+      confirmed == Satoshi(100000000L)
+    }, max = 30 seconds, interval = 1 second)
+
+    probe.send(wallet, GetCurrentReceiveAddress)
+    val GetCurrentReceiveAddressResponse(address1) = probe.expectMsgType[GetCurrentReceiveAddressResponse]
+
+    logger.info(s"sending 1 btc to $address1")
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address1 :: 1.0 :: Nil))
+    probe.expectMsgType[JValue]
+    logger.info(s"sending 0.5 btc to $address1")
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address1 :: 0.5 :: Nil))
+    probe.expectMsgType[JValue]
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+      confirmed == Satoshi(250000000L)
+    }, max = 30 seconds, interval = 1 second)
+  }
+
+  test("receive 'confidence changed' notification") {
+    val probe = TestProbe()
+    val listener = TestProbe()
+
+    listener.send(wallet, AddStatusListener(listener.ref))
+
+    probe.send(wallet, GetCurrentReceiveAddress)
+    val GetCurrentReceiveAddressResponse(address) = probe.expectMsgType[GetCurrentReceiveAddressResponse]
+
+    probe.send(wallet, GetBalance)
+    val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+
+    logger.info(s"sending 1 btc to $address")
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address :: 1.0 :: Nil))
+    val JString(txid) = probe.expectMsgType[JValue]
+    logger.info(s"$txid send 1 btc to us at $address")
+
+    val TransactionReceived(tx, 0, received, sent, _) = listener.receiveOne(5 seconds)
+    assert(tx.txid === BinaryData(txid))
+    assert(received === Satoshi(100000000))
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      confirmed1 - confirmed == Satoshi(100000000L)
+    }, max = 30 seconds, interval = 1 second)
+
+    awaitCond({
+      val msg = listener.receiveOne(5 seconds)
+      msg == TransactionConfidenceChanged(BinaryData(txid), 1)
+    }, max = 30 seconds, interval = 1 second)
+  }
+
+  test("send money to someone else (we broadcast)") {
+    val probe = TestProbe()
+    probe.send(bitcoincli, BitcoinReq("getnewaddress"))
+    val JString(address) = probe.expectMsgType[JValue]
+    val (Base58.Prefix.PubkeyAddressTestnet, pubKeyHash) = Base58Check.decode(address)
+
+    probe.send(wallet, GetBalance)
+    val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+
+    // create a tx that sends money to Bitcoin Core's address
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(1 btc, Script.pay2pkh(pubKeyHash)) :: Nil, lockTime = 0L)
+    probe.send(wallet, CompleteTransaction(tx, 20000))
+    val CompleteTransactionResponse(tx1, None) = probe.expectMsgType[CompleteTransactionResponse]
+
+    // send it ourselves
+    logger.info(s"sending 1 btc to $address with tx ${tx1.txid}")
+    probe.send(wallet, BroadcastTransaction(tx1))
+    val BroadcastTransactionResponse(_, None) = probe.expectMsgType[BroadcastTransactionResponse]
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(bitcoincli, BitcoinReq("getreceivedbyaddress", address :: Nil))
+      val JDouble(value) = probe.expectMsgType[JValue]
+      value == 1.0
+    }, max = 30 seconds, interval = 1 second)
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 < confirmed - Btc(1) && confirmed1 > confirmed - Btc(1) - Satoshi(50000)
+    }, max = 30 seconds, interval = 1 second)
+  }
+
+  test("send money to ourselves (we broadcast)") {
+    val probe = TestProbe()
+    probe.send(wallet, GetCurrentReceiveAddress)
+    val GetCurrentReceiveAddressResponse(address) = probe.expectMsgType[GetCurrentReceiveAddressResponse]
+    val (Base58.Prefix.ScriptAddressTestnet, scriptHash) = Base58Check.decode(address)
+
+    probe.send(wallet, GetBalance)
+    val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+
+    // create a tx that sends money to Bitcoin Core's address
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(1 btc, OP_HASH160 :: OP_PUSHDATA(scriptHash) :: OP_EQUAL :: Nil) :: Nil, lockTime = 0L)
+    probe.send(wallet, CompleteTransaction(tx, 20000))
+    val CompleteTransactionResponse(tx1, None) = probe.expectMsgType[CompleteTransactionResponse]
+
+    // send it ourselves
+    logger.info(s"sending 1 btc to $address with tx ${tx1.txid}")
+    probe.send(wallet, BroadcastTransaction(tx1))
+    val BroadcastTransactionResponse(_, None) = probe.expectMsgType[BroadcastTransactionResponse]
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 < confirmed && confirmed1 > confirmed - Satoshi(50000)
+    }, max = 30 seconds, interval = 1 second)
+  }
+
+  ignore("handle reorgs (pending receive)") {
+    val probe = TestProbe()
+
+    probe.send(wallet, GetBalance)
+    val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+
+    probe.send(wallet, GetCurrentReceiveAddress)
+    val GetCurrentReceiveAddressResponse(address) = probe.expectMsgType[GetCurrentReceiveAddressResponse]
+
+    // send money to our receive address
+    logger.info(s"sending 0.7 btc to $address")
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address :: 0.7 :: Nil))
+    probe.expectMsgType[JValue]
+
+    // generate 1 block
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    val JArray(List(JString(blockId))) = probe.expectMsgType[JValue]
+
+    // wait until our balance has been updated
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 == confirmed + Btc(0.7)
+    }, max = 30 seconds, interval = 1 second)
+
+    // now invalidate the last block
+    probe.send(bitcoincli, BitcoinReq("invalidateblock", blockId :: Nil))
+    probe.expectMsgType[JValue]
+
+    // and restart bitcoind, which should remove pending wallet txs
+    // bitcoind was started with -zapwallettxes=2
+    stopBitcoind
+    Thread.sleep(2000)
+    startBitcoind
+    Thread.sleep(2000)
+
+
+    // generate 2 new blocks. the tx that sent us money is no longer there,
+    // the corresponding utxo should have been removed and our balance should
+    // be back to what it was before
+    probe.send(bitcoincli, BitcoinReq("generate", 2 :: Nil))
+    probe.expectMsgType[JValue]
+    val reorg = s"$PATH_ELECTRUMX/electrumx_rpc.py reorg 2".!!
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 == confirmed
+    }, max = 30 seconds, interval = 1 second)
+  }
+
+  ignore("handle reorgs (pending send)") {
+    val probe = TestProbe()
+    probe.send(bitcoincli, BitcoinReq("getnewaddress"))
+    val JString(address) = probe.expectMsgType[JValue]
+    val (Base58.Prefix.PubkeyAddressTestnet, pubKeyHash) = Base58Check.decode(address)
+
+    probe.send(wallet, GetBalance)
+    val GetBalanceResponse(confirmed, unconfirmed) = probe.expectMsgType[GetBalanceResponse]
+    logger.debug(s"we start with a balance of $confirmed")
+
+    // create a tx that sends money to Bitcoin Core's address
+    val amount = 0.5 btc
+    val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(amount, Script.pay2pkh(pubKeyHash)) :: Nil, lockTime = 0L)
+    probe.send(wallet, CompleteTransaction(tx, 20000))
+    val CompleteTransactionResponse(tx1, None) = probe.expectMsgType[CompleteTransactionResponse]
+
+    // send it ourselves
+    logger.info(s"sending $amount to $address with tx ${tx1.txid}")
+    probe.send(wallet, BroadcastTransaction(tx1))
+    val BroadcastTransactionResponse(_, None) = probe.expectMsgType[BroadcastTransactionResponse]
+
+    probe.send(bitcoincli, BitcoinReq("generate", 1 :: Nil))
+    val JArray(List(JString(blockId))) = probe.expectMsgType[JValue]
+
+    awaitCond({
+      probe.send(bitcoincli, BitcoinReq("getreceivedbyaddress", address :: Nil))
+      val JDouble(value) = probe.expectMsgType[JValue]
+      value == amount.amount.toDouble
+    }, max = 30 seconds, interval = 1 second)
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 < confirmed - amount && confirmed1 > confirmed - amount - Satoshi(50000)
+    }, max = 30 seconds, interval = 1 second)
+
+    // now invalidate the last block
+    probe.send(bitcoincli, BitcoinReq("getblockcount"))
+    val JInt(count) = probe.expectMsgType[JValue]
+    probe.send(bitcoincli, BitcoinReq("invalidateblock", blockId :: Nil))
+    val foo = probe.expectMsgType[JValue]
+    probe.send(bitcoincli, BitcoinReq("getblockcount"))
+    val JInt(count1) = probe.expectMsgType[JValue]
+
+    // and restart bitcoind, which should remove pending wallet txs
+    // bitcoind was started with -zapwallettxes=2
+    stopBitcoind
+    Thread.sleep(2000)
+    startBitcoind
+    Thread.sleep(2000)
+
+    // generate 2 new blocks. the tx that sent us money is no longer there,
+    // the corresponding utxo should have been removed and our balance should
+    // be back to what it was before
+    probe.send(bitcoincli, BitcoinReq("generate", 2 :: Nil))
+    probe.expectMsgType[JValue]
+    val reorg = s"$PATH_ELECTRUMX/electrumx_rpc.py reorg 2".!!
+
+    awaitCond({
+      probe.send(wallet, GetBalance)
+      val GetBalanceResponse(confirmed1, unconfirmed1) = probe.expectMsgType[GetBalanceResponse]
+      logger.debug(s"current balance is $confirmed1")
+      confirmed1 == confirmed
+    }, max = 30 seconds, interval = 1 second)
+  }
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWatcherSpec.scala

@@ -0,0 +1,82 @@
+package fr.acinq.eclair.blockchain.electrum
+
+import akka.actor.Props
+import akka.testkit.TestProbe
+import fr.acinq.bitcoin.Crypto.PrivateKey
+import fr.acinq.bitcoin.{Base58, OP_PUSHDATA, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptFlags, SigVersion, Transaction, TxIn, TxOut}
+import fr.acinq.eclair.blockchain.{WatchConfirmed, WatchEventConfirmed, WatchEventSpent, WatchSpent}
+import fr.acinq.eclair.channel.{BITCOIN_FUNDING_DEPTHOK, BITCOIN_FUNDING_SPENT}
+import org.json4s.JsonAST.{JArray, JString, JValue}
+
+import scala.concurrent.duration._
+
+class ElectrumWatcherSpec extends IntegrationSpec {
+  test("watch for confirmed transactions") {
+    val probe = TestProbe()
+    val watcher = system.actorOf(Props(new ElectrumWatcher(electrumClient)))
+
+    probe.send(bitcoincli, BitcoinReq("getnewaddress"))
+    val JString(address) = probe.expectMsgType[JValue]
+    println(address)
+
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address :: 1.0 :: Nil))
+    val JString(txid) = probe.expectMsgType[JValue](3000 seconds)
+
+    probe.send(bitcoincli, BitcoinReq("getrawtransaction", txid :: Nil))
+    val JString(hex) = probe.expectMsgType[JValue]
+    val tx = Transaction.read(hex)
+
+    val listener = TestProbe()
+    probe.send(watcher, WatchConfirmed(listener.ref, tx.txid, tx.txOut(0).publicKeyScript, 4, BITCOIN_FUNDING_DEPTHOK))
+    probe.send(bitcoincli, BitcoinReq("generate", 3 :: Nil))
+    listener.expectNoMsg(1 second)
+    probe.send(bitcoincli, BitcoinReq("generate", 2 :: Nil))
+    val confirmed = listener.expectMsgType[WatchEventConfirmed](20 seconds)
+    system.stop(watcher)
+  }
+
+  test("watch for spent transactions") {
+    val probe = TestProbe()
+    val watcher = system.actorOf(Props(new ElectrumWatcher(electrumClient)))
+
+    probe.send(bitcoincli, BitcoinReq("getnewaddress"))
+    val JString(address) = probe.expectMsgType[JValue]
+    println(address)
+
+    probe.send(bitcoincli, BitcoinReq("dumpprivkey", address :: Nil))
+    val JString(wif) = probe.expectMsgType[JValue]
+    val priv = PrivateKey.fromBase58(wif, Base58.Prefix.SecretKeyTestnet)
+
+    probe.send(bitcoincli, BitcoinReq("sendtoaddress", address :: 1.0 :: Nil))
+    val JString(txid) = probe.expectMsgType[JValue](3000 seconds)
+
+    probe.send(bitcoincli, BitcoinReq("getrawtransaction", txid :: Nil))
+    val JString(hex) = probe.expectMsgType[JValue]
+    val tx = Transaction.read(hex)
+
+    // find the output for the address we generated and create a tx that spends it
+    val pos = tx.txOut.indexWhere(_.publicKeyScript == Script.write(Script.pay2pkh(priv.publicKey)))
+    assert(pos != -1)
+    val spendingTx = {
+      val tmp = Transaction(version = 2,
+        txIn = TxIn(OutPoint(tx, pos), signatureScript = Nil, sequence = TxIn.SEQUENCE_FINAL) :: Nil,
+        txOut = TxOut(tx.txOut(pos).amount - Satoshi(1000), publicKeyScript = Script.pay2pkh(priv.publicKey)) :: Nil,
+        lockTime = 0)
+      val sig = Transaction.signInput(tmp, 0, tx.txOut(pos).publicKeyScript, SIGHASH_ALL, tx.txOut(pos).amount, SigVersion.SIGVERSION_BASE, priv)
+      val signedTx = tmp.updateSigScript(0, OP_PUSHDATA(sig) :: OP_PUSHDATA(priv.publicKey.toBin) :: Nil)
+      Transaction.correctlySpends(signedTx, Seq(tx), ScriptFlags.STANDARD_SCRIPT_VERIFY_FLAGS)
+      signedTx
+    }
+
+    val listener = TestProbe()
+    probe.send(watcher, WatchSpent(listener.ref, tx.txid, pos, tx.txOut(pos).publicKeyScript, BITCOIN_FUNDING_SPENT))
+    listener.expectNoMsg(1 second)
+    probe.send(bitcoincli, BitcoinReq("sendrawtransaction", Transaction.write(spendingTx).toString :: Nil))
+    probe.expectMsgType[JValue]
+    probe.send(bitcoincli, BitcoinReq("generate", 2 :: Nil))
+    val blocks = probe.expectMsgType[JValue]
+    val JArray(List(JString(block1), JString(block2))) = blocks
+    val spent = listener.expectMsgType[WatchEventSpent](20 seconds)
+    system.stop(watcher)
+  }
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/FallbackFeeProviderSpec.scala

@@ -0,0 +1,62 @@
+package fr.acinq.eclair.blockchain.fee
+
+import org.junit.runner.RunWith
+import org.scalatest.FunSuite
+import org.scalatest.junit.JUnitRunner
+
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Future}
+import scala.util.Random
+
+@RunWith(classOf[JUnitRunner])
+class FallbackFeeProviderSpec extends FunSuite {
+
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  /**
+    * This provider returns a constant value, but fails after ttl tries
+    *
+    * @param ttl
+    * @param feeratesPerByte
+    */
+  class FailingFeeProvider(ttl: Int, val feeratesPerByte: FeeratesPerByte) extends FeeProvider {
+    var i = 0
+
+    override def getFeerates: Future[FeeratesPerByte] =
+      if (i < ttl) {
+        i = i + 1
+        Future.successful(feeratesPerByte)
+      } else Future.failed(new RuntimeException())
+  }
+
+  def dummyFeerates = FeeratesPerByte(Random.nextInt(10000), Random.nextInt(10000), Random.nextInt(10000), Random.nextInt(10000), Random.nextInt(10000), Random.nextInt(10000))
+
+  def await[T](f: Future[T]): T = Await.result(f, 3 seconds)
+
+  test("fee provider failover") {
+    val provider0 = new FailingFeeProvider(-1, dummyFeerates) // always fails
+    val provider1 = new FailingFeeProvider(1, dummyFeerates) // fails after 1 try
+    val provider3 = new FailingFeeProvider(3, dummyFeerates) // fails after 3 tries
+    val provider5 = new FailingFeeProvider(5, dummyFeerates) // fails after 5 tries
+    val provider7 = new FailingFeeProvider(Int.MaxValue, dummyFeerates) // "never" fails
+
+    val fallbackFeeProvider = new FallbackFeeProvider(provider0 :: provider1 :: provider3 :: provider5 :: provider7 :: Nil)
+
+    assert(await(fallbackFeeProvider.getFeerates) === provider1.feeratesPerByte)
+
+    assert(await(fallbackFeeProvider.getFeerates) === provider3.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider3.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider3.feeratesPerByte)
+
+    assert(await(fallbackFeeProvider.getFeerates) === provider5.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider5.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider5.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider5.feeratesPerByte)
+    assert(await(fallbackFeeProvider.getFeerates) === provider5.feeratesPerByte)
+
+    assert(await(fallbackFeeProvider.getFeerates) === provider7.feeratesPerByte)
+
+  }
+
+
+}

eclair-core/src/test/scala/fr/acinq/eclair/channel/FuzzyPipe.scala

@@ -1,8 +1,7 @@
-package fr.acinq.eclair.channel.states
+package fr.acinq.eclair.channel
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Stash}
 import fr.acinq.eclair.channel.Commitments.msg2String
-import fr.acinq.eclair.channel.{INPUT_DISCONNECTED, INPUT_RECONNECTED}
 import fr.acinq.eclair.wire.LightningMessage
 
 import scala.concurrent.duration._
@@ -11,7 +10,7 @@ import scala.util.Random
 /**
   * A Fuzzy [[fr.acinq.eclair.Pipe]] which randomly disconnects/reconnects peers.
   */
-class FuzzyPipe extends Actor with Stash with ActorLogging {
+class FuzzyPipe(fuzzy: Boolean) extends Actor with Stash with ActorLogging {
 
   import scala.concurrent.ExecutionContext.Implicits.global
 
@@ -24,7 +23,8 @@ class FuzzyPipe extends Actor with Stash with ActorLogging {
   }
 
   def stayOrDisconnect(a: ActorRef, b: ActorRef, countdown: Int) = {
-    if (countdown > 1) context become connected(a, b, countdown - 1)
+    if (!fuzzy) context become connected(a, b, countdown - 1) // fuzzy mode disabled, we never disconnect
+    else if (countdown > 1) context become connected(a, b, countdown - 1)
     else {
       log.debug("DISCONNECTED")
       a ! INPUT_DISCONNECTED
@@ -56,6 +56,6 @@ class FuzzyPipe extends Actor with Stash with ActorLogging {
       log.debug("RECONNECTED")
       a ! INPUT_RECONNECTED(self)
       b ! INPUT_RECONNECTED(self)
-      context become connected(a, b, Random.nextInt(20))
+      context become connected(a, b, Random.nextInt(40))
   }
 }

eclair-core/src/test/scala/fr/acinq/eclair/channel/FuzzySpec.scala

@@ -1,4 +1,4 @@
-package fr.acinq.eclair.channel.states
+package fr.acinq.eclair.channel
 
 import java.util.concurrent.CountDownLatch
 
@@ -9,12 +9,12 @@ import fr.acinq.bitcoin.{BinaryData, MilliSatoshi}
 import fr.acinq.eclair.TestConstants.{Alice, Bob}
 import fr.acinq.eclair._
 import fr.acinq.eclair.blockchain._
-import fr.acinq.eclair.channel.{Data, State, _}
+import fr.acinq.eclair.channel.states.StateTestsHelperMethods
 import fr.acinq.eclair.payment._
-import fr.acinq.eclair.router.Hop
 import fr.acinq.eclair.wire._
 import grizzled.slf4j.Logging
 import org.junit.runner.RunWith
+import org.scalatest.Tag
 import org.scalatest.junit.JUnitRunner
 
 import scala.collection.immutable.Nil
@@ -30,13 +30,16 @@ class FuzzySpec extends TestkitBaseClass with StateTestsHelperMethods with Loggi
   type FixtureParam = Tuple7[TestFSMRef[State, Data, Channel], TestFSMRef[State, Data, Channel], ActorRef, ActorRef, ActorRef, ActorRef, ActorRef]
 
   override def withFixture(test: OneArgTest) = {
-    val pipe = system.actorOf(Props(new FuzzyPipe()))
+    val fuzzy = test.tags.contains("fuzzy")
+    val pipe = system.actorOf(Props(new FuzzyPipe(fuzzy)))
     val alice2blockchain = TestProbe()
     val bob2blockchain = TestProbe()
-    val paymentHandlerA = system.actorOf(Props(new LocalPaymentHandler(Alice.nodeParams)), name = "payment-handler-a")
-    val paymentHandlerB = system.actorOf(Props(new LocalPaymentHandler(Bob.nodeParams)), name = "payment-handler-b")
-    val relayerA = system.actorOf(Relayer.props(Alice.nodeParams.privateKey, paymentHandlerA), "relayer-a")
-    val relayerB = system.actorOf(Relayer.props(Bob.nodeParams.privateKey, paymentHandlerB), "relayer-b")
+    val paymentHandlerA = system.actorOf(Props(new LocalPaymentHandler(Alice.nodeParams)))
+    val paymentHandlerB = system.actorOf(Props(new LocalPaymentHandler(Bob.nodeParams)))
+    val registerA = TestProbe()
+    val registerB = TestProbe()
+    val relayerA = system.actorOf(Relayer.props(Alice.nodeParams, registerA.ref, paymentHandlerA))
+    val relayerB = system.actorOf(Relayer.props(Bob.nodeParams, registerB.ref, paymentHandlerB))
     val router = TestProbe()
     val wallet = new TestWallet
     val alice: TestFSMRef[State, Data, Channel] = TestFSMRef(new Channel(Alice.nodeParams, wallet, Bob.id, alice2blockchain.ref, router.ref, relayerA))
@@ -52,7 +55,6 @@ class FuzzySpec extends TestkitBaseClass with StateTestsHelperMethods with Loggi
       pipe ! (alice, bob)
       alice2blockchain.expectMsgType[WatchSpent]
       alice2blockchain.expectMsgType[WatchConfirmed]
-      alice2blockchain.expectMsgType[PublishAsap]
       bob2blockchain.expectMsgType[WatchSpent]
       bob2blockchain.expectMsgType[WatchConfirmed]
       alice ! WatchEventConfirmed(BITCOIN_FUNDING_DEPTHOK, 400000, 42)
@@ -67,57 +69,53 @@ class FuzzySpec extends TestkitBaseClass with StateTestsHelperMethods with Loggi
 
   class SenderActor(channel: TestFSMRef[State, Data, Channel], paymentHandler: ActorRef, latch: CountDownLatch) extends Actor with ActorLogging {
 
-    /*if (channel.stateData.asInstanceOf[DATA_NORMAL].commitments.localCommit.spec.htlcs.size >= 10 || channel.stateData.asInstanceOf[DATA_NORMAL].commitments.remoteCommit.spec.htlcs.size >= 10) {
-      context stop self
-    } else {
-
-    }*/
-
     // we don't want to be below htlcMinimumMsat
     val requiredAmount = 1000000
 
     def buildCmdAdd(paymentHash: BinaryData, dest: PublicKey) = {
       // allow overpaying (no more than 2 times the required amount)
       val amount = requiredAmount + Random.nextInt(requiredAmount)
-      val expiry = Globals.blockCount.get().toInt + PaymentLifecycle.defaultHtlcExpiry
+      val expiry = Globals.blockCount.get().toInt + PaymentLifecycle.defaultMinFinalCltvExpiry
       PaymentLifecycle.buildCommand(amount, expiry, paymentHash, Hop(null, dest, null) :: Nil)._1
     }
 
-    def initiatePayment = {
-      paymentHandler ! ReceivePayment(MilliSatoshi(requiredAmount), "One coffee")
-      context become waitingForPaymentRequest
-    }
+    def initiatePayment(stopping: Boolean) =
+      if (stopping) {
+        context stop self
+      } else {
+        paymentHandler ! ReceivePayment(MilliSatoshi(requiredAmount), "One coffee")
+        context become waitingForPaymentRequest
+      }
 
-    initiatePayment
+    initiatePayment(false)
 
     override def receive: Receive = ???
 
     def waitingForPaymentRequest: Receive = {
       case req: PaymentRequest =>
         channel ! buildCmdAdd(req.paymentHash, req.nodeId)
-        context become waitingForFulfill
+        context become waitingForFulfill(false)
     }
 
-    def waitingForFulfill: Receive = {
+    def waitingForFulfill(stopping: Boolean): Receive = {
       case u: UpdateFulfillHtlc =>
         log.info(s"successfully sent htlc #${u.id}")
         latch.countDown()
-        initiatePayment
+        initiatePayment(stopping)
       case u: UpdateFailHtlc =>
         log.warning(s"htlc failed: ${u.id}")
-        initiatePayment
+        initiatePayment(stopping)
       case Status.Failure(t) =>
         log.error(s"htlc error: ${t.getMessage}")
-        initiatePayment
-      case 'cancelled =>
-        log.warning(s"our htlc was cancelled!")
-        // htlc was dropped because of a disconnection
-        initiatePayment
+        initiatePayment(stopping)
+      case 'stop =>
+        log.warning(s"stopping...")
+        context become waitingForFulfill(true)
     }
 
   }
-  
-  test("fuzzy test with only one party sending HTLCs") {
+
+  test("fuzzy test with only one party sending HTLCs", Tag("fuzzy")) {
     case (alice, bob, _, _, _, _, paymentHandlerB) =>
       val latch = new CountDownLatch(100)
       system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch)))
@@ -125,9 +123,9 @@ class FuzzySpec extends TestkitBaseClass with StateTestsHelperMethods with Loggi
       awaitCond(latch.getCount == 0, max = 2 minutes)
       assert(alice.stateName == NORMAL || alice.stateName == OFFLINE)
       assert(bob.stateName == NORMAL || alice.stateName == OFFLINE)
-    }
+  }
 
-  test("fuzzy test with both parties sending HTLCs") {
+  test("fuzzy test with both parties sending HTLCs", Tag("fuzzy")) {
     case (alice, bob, _, _, _, paymentHandlerA, paymentHandlerB) =>
       val latch = new CountDownLatch(100)
       system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch)))
@@ -139,4 +137,43 @@ class FuzzySpec extends TestkitBaseClass with StateTestsHelperMethods with Loggi
       assert(bob.stateName == NORMAL || alice.stateName == OFFLINE)
   }
 
+  test("one party sends lots of htlcs send shutdown") {
+    case (alice, _, _, _, _, _, paymentHandlerB) =>
+      val latch = new CountDownLatch(20)
+      val senders = system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch))) ::
+        system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch))) ::
+        system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch))) :: Nil
+      awaitCond(latch.getCount == 0, max = 2 minutes)
+      val sender = TestProbe()
+      awaitCond({
+        sender.send(alice, CMD_CLOSE(None))
+        sender.expectMsgAnyClassOf(classOf[String], classOf[Status.Failure]) == "ok"
+      }, max = 30 seconds)
+      senders.foreach(_ ! 'stop)
+      awaitCond(alice.stateName == CLOSING)
+      awaitCond(alice.stateName == CLOSING)
+  }
+
+  test("both parties send lots of htlcs send shutdown") {
+    case (alice, bob, _, _, _, paymentHandlerA, paymentHandlerB) =>
+      val latch = new CountDownLatch(30)
+      val senders = system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch))) ::
+        system.actorOf(Props(new SenderActor(alice, paymentHandlerB, latch))) ::
+        system.actorOf(Props(new SenderActor(bob, paymentHandlerA, latch))) ::
+        system.actorOf(Props(new SenderActor(bob, paymentHandlerA, latch))) :: Nil
+      awaitCond(latch.getCount == 0, max = 2 minutes)
+      val sender = TestProbe()
+      awaitCond({
+        sender.send(alice, CMD_CLOSE(None))
+        val resa = sender.expectMsgAnyClassOf(classOf[String], classOf[Status.Failure])
+        sender.send(bob, CMD_CLOSE(None))
+        val resb = sender.expectMsgAnyClassOf(classOf[String], classOf[Status.Failure])
+        // we only need that one of them succeeds
+        resa == "ok" || resb == "ok"
+      }, max = 30 seconds)
+      senders.foreach(_ ! 'stop)
+      awaitCond(alice.stateName == CLOSING)
+      awaitCond(alice.stateName == CLOSING)
+  }
+
 }

eclair-core/src/test/scala/fr/acinq/eclair/channel/ThroughputSpec.scala

@@ -4,10 +4,12 @@ import java.util.concurrent.CountDownLatch
 import java.util.concurrent.atomic.AtomicLong
 
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
+import akka.testkit.TestProbe
 import fr.acinq.bitcoin.{BinaryData, Crypto}
 import fr.acinq.eclair.TestConstants.{Alice, Bob}
 import fr.acinq.eclair._
 import fr.acinq.eclair.blockchain._
+import fr.acinq.eclair.blockchain.bitcoind.ZmqWatcher
 import fr.acinq.eclair.payment.Relayer
 import fr.acinq.eclair.wire.{Init, UpdateAddHtlc}
 import org.junit.runner.RunWith
@@ -52,8 +54,10 @@ class ThroughputSpec extends FunSuite {
           context.become(run(h2r - htlc.paymentHash))
       }
     }), "payment-handler")
-    val relayerA = system.actorOf(Relayer.props(Alice.nodeParams.privateKey, paymentHandler))
-    val relayerB = system.actorOf(Relayer.props(Bob.nodeParams.privateKey, paymentHandler))
+    val registerA = TestProbe()
+    val registerB = TestProbe()
+    val relayerA = system.actorOf(Relayer.props(Alice.nodeParams, registerA.ref, paymentHandler))
+    val relayerB = system.actorOf(Relayer.props(Bob.nodeParams, registerB.ref, paymentHandler))
     val wallet = new TestWallet
     val alice = system.actorOf(Channel.props(Alice.nodeParams, wallet, Bob.id, blockchain, ???, relayerA), "a")
     val bob = system.actorOf(Channel.props(Bob.nodeParams, wallet, Alice.id, blockchain, ???, relayerB), "b")

eclair-core/src/test/scala/fr/acinq/eclair/channel/states/StateTestsHelperMethods.scala

@@ -4,10 +4,10 @@ import akka.testkit.{TestFSMRef, TestKitBase, TestProbe}
 import fr.acinq.bitcoin.{BinaryData, Crypto}
 import fr.acinq.eclair.TestConstants.{Alice, Bob}
 import fr.acinq.eclair.blockchain._
+import fr.acinq.eclair.blockchain.fee.FeeratesPerKw
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.Sphinx
-import fr.acinq.eclair.payment.PaymentLifecycle
-import fr.acinq.eclair.router.Hop
+import fr.acinq.eclair.payment.{Hop, PaymentLifecycle}
 import fr.acinq.eclair.wire._
 import fr.acinq.eclair.{Globals, TestConstants}
 
@@ -30,7 +30,7 @@ trait StateTestsHelperMethods extends TestKitBase {
                    relayer: TestProbe)
 
   def init(): Setup = {
-    Globals.feeratePerKw.set(TestConstants.feeratePerKw)
+    Globals.feeratesPerKw.set(FeeratesPerKw.single(TestConstants.feeratePerKw))
     val alice2bob = TestProbe()
     val bob2alice = TestProbe()
     val alice2blockchain = TestProbe()
@@ -69,7 +69,6 @@ trait StateTestsHelperMethods extends TestKitBase {
     bob2alice.forward(alice)
     alice2blockchain.expectMsgType[WatchSpent]
     alice2blockchain.expectMsgType[WatchConfirmed]
-    alice2blockchain.expectMsgType[PublishAsap]
     bob2blockchain.expectMsgType[WatchSpent]
     bob2blockchain.expectMsgType[WatchConfirmed]
     alice ! WatchEventConfirmed(BITCOIN_FUNDING_DEPTHOK, 400000, 42)

eclair-core/src/test/scala/fr/acinq/eclair/channel/states/a/WaitForAcceptChannelStateSpec.scala

@@ -1,6 +1,5 @@
 package fr.acinq.eclair.channel.states.a
 
-import akka.actor.ActorRef
 import akka.testkit.{TestFSMRef, TestProbe}
 import fr.acinq.eclair.TestConstants.{Alice, Bob}
 import fr.acinq.eclair.channel.states.StateTestsHelperMethods
@@ -50,22 +49,11 @@ class WaitForAcceptChannelStateSpec extends TestkitBaseClass with StateTestsHelp
       val reserveTooHigh = (0.3 * TestConstants.fundingSatoshis).toLong
       alice ! accept.copy(channelReserveSatoshis = reserveTooHigh)
       val error = alice2bob.expectMsgType[Error]
-      assert(new String(error.data) === "requirement failed: channelReserveSatoshis too high: ratio=0.3 max=0.05")
+      assert(new String(error.data) === "channelReserveSatoshis too high: reserve=300000 fundingRatio=0.3 maxFundingRatio=0.05")
       awaitCond(alice.stateName == CLOSED)
     }
   }
 
-  /*test("recv funding tx") { case (alice, alice2bob, bob2alice, alice2blockchain, blockchain) =>
-    within(30 seconds) {
-      bob2alice.expectMsgType[OpenChannel]
-      bob2alice.forward(alice)
-      alice2blockchain.expectMsgType[MakeFundingTx]
-      alice2blockchain.forward(blockchain)
-      awaitCond(alice.stateName == WAIT_FOR_FUNDING_SIGNED)
-      alice2bob.expectMsgType[OpenChannel]
-    }
-  }*/
-
   test("recv Error") { case (bob, alice2bob, bob2alice, _) =>
     within(30 seconds) {
       bob ! Error("00" * 32, "oops".getBytes)