inital commit, which is clearly not initial

Signed-off-by: Uncle Stretch <uncle.stretch@ghostchain.io>

cli/Cargo.toml

@@ -0,0 +1,81 @@
+[package]
+name = "ghost-cli"
+description = "Ghost Client Node"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+
+[package.metadata.wasm-pack.profile.release]
+# `wasm-opt` has some problems on Linux, see
+# https://github.com/rustwasm/wasm-pack/issues/781 etc.
+wasm-opt = false
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+cfg-if = { workspace = true }
+clap = { workspace = true, features = ["derive"], optional = true }
+log = { workspace = true, default-features = false }
+serde_json = { workspace = true, default-features = false }
+thiserror = { workspace = true }
+futures = { workspace = true }
+pyro = { workspace = true, optional = true }
+pyroscope_pprofrs = { workspace = true, optional = true }
+
+service = { workspace = true, optional = true }
+ghost-client-cli = { workspace = true, optional = true }
+ghost-machine-primitives = { workspace = true, optional = true }
+
+sp-core = { workspace = true, default-features = true }
+sp-io = { workspace = true, default-features = true }
+sp-runtime = { workspace = true, default-features = true }
+sp-maybe-compressed-blob = { workspace = true, default-features = true }
+keyring = { workspace = true, default-features = true }
+frame-benchmarking-cli = { workspace = true, default-features = true, optional = true }
+sc-cli = { workspace = true, default-features = true, optional = true }
+sc-service = { workspace = true, default-features = true, optional = true }
+ghost-metrics = { workspace = true, default-features = true }
+primitives = { workspace = true, default-features = true }
+sc-tracing = { workspace = true, default-features = true, optional = true }
+sc-sysinfo = { workspace = true, default-features = true }
+sc-executor = { workspace = true, default-features = true }
+sc-storage-monitor = { workspace = true, default-features = true }
+
+[build-dependencies]
+substrate-build-script-utils = { workspace = true }
+
+[features]
+default = ["cli", "db", "full-node"]
+db = ["service/db"]
+cli = [
+	"clap",
+    "frame-benchmarking-cli",
+	"sc-cli",
+	"sc-service",
+	"sc-tracing",
+	"service",
+	"ghost-client-cli",
+	"ghost-machine-primitives",
+]
+runtime-benchmarks = [
+    "frame-benchmarking-cli?/runtime-benchmarks",
+    "ghost-metrics/runtime-benchmarks",
+    "sc-service?/runtime-benchmarks",
+    "service/runtime-benchmarks",
+    "sp-runtime/runtime-benchmarks",
+]
+full-node = ["service/full-node"]
+try-runtime = [
+    "service/try-runtime",
+    "sp-runtime/try-runtime",
+]
+fast-runtime = ["service/fast-runtime"]
+pyroscope = ["pyro", "pyroscope_pprofrs"]
+
+# Configure the native runtimes to use
+ghost-native = []
+casper-native = ["service/casper-native"]

cli/build.rs

@@ -0,0 +1,7 @@
+fn main () {
+    if let Ok(profile) = std::env::var("PROFILE") {
+        println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
+    }
+    substrate_build_script_utils::generate_cargo_keys();
+    substrate_build_script_utils::rerun_if_git_head_changed();
+}

cli/src/cli.rs

@@ -0,0 +1,112 @@
+//! Ghost CLI library.
+
+use clap::Parser;
+use ghost_client_cli::commands::{KeySubcommand, VanityCmd};
+
+#[derive(Debug, clap::Subcommand)]
+pub enum KeyCmd {
+    /// Key utility for the CLI
+    #[clap(flatten)]
+    KeyCli(KeySubcommand),
+
+    /// Sign a message, with a given (secret) key.
+    Sign(sc_cli::SignCmd),
+
+    /// Generate a seed that provides a vanity address/
+    Vanity(VanityCmd),
+
+    /// Verify a signature for a mesage, provided on STDIN, with a given
+    /// (public or secret) key.
+    Verify(sc_cli::VerifyCmd),
+}
+
+impl KeyCmd {
+    pub fn run<C: sc_cli::SubstrateCli>(&self, cli: &C) -> Result<(), sc_cli::Error> {
+        match self {
+            KeyCmd::KeyCli(cmd) => cmd.run(cli),
+            KeyCmd::Sign(cmd) => cmd.run(),
+            KeyCmd::Vanity(cmd) => cmd.run(),
+            KeyCmd::Verify(cmd) => cmd.run(),
+        }
+    }
+}
+
+/// Possible subcommands of the main binary.
+#[derive(Debug, clap::Subcommand)]
+pub enum Subcommand {
+    /// Build a chain specification.
+    BuildSpec(sc_cli::BuildSpecCmd),
+
+    /// Validate blocks.
+    CheckBlock(sc_cli::CheckBlockCmd),
+
+    /// Export blocks.
+    ExportBlocks(sc_cli::ExportBlocksCmd),
+
+    /// Export the state of a given block into a chain spec.
+    ExportState(sc_cli::ExportStateCmd),
+
+    /// Import blocks.
+    ImportBlocks(sc_cli::ImportBlocksCmd),
+
+    /// Remove the whole chain.
+    PurgeChain(sc_cli::PurgeChainCmd),
+
+    /// Revert the chain to a previous state.
+    Revert(sc_cli::RevertCmd),
+
+    /// The custom benchmark subcommmand benchmarking runtime pallets.
+    #[clap(subcommand)]
+    Benchmark(frame_benchmarking_cli::BenchmarkCmd),
+
+    /// Key managment cli utilities.
+    #[clap(subcommand)]
+    Key(KeyCmd),
+
+    /// Db meta columns information
+    ChainInfo(sc_cli::ChainInfoCmd),
+}
+
+#[allow(missing_docs)]
+#[derive(Debug, Parser)]
+#[group(skip)]
+pub struct RunCmd {
+    #[clap(flatten)]
+    pub base: sc_cli::RunCmd,
+
+    /// Disable autimatic hardware benchmarks.
+    ///
+    /// By default these benchamrks are automatically ran at startup and
+    /// measure the CPU speed, the memory bandwidth and the disk speed.
+    ///
+    /// The results, are then printed out in the logs, and also send as part
+    /// of telemetry, if telemetry is enabled.
+    #[arg(long)]
+    pub no_hardware_benchmarks: bool,
+
+    /// Enable the block authoring backoff that is triggered when finality is 
+    /// lagging.
+    #[arg(long)]
+    pub force_authoring_backoff: bool,
+
+    /// Add the destination address to the `pyroscope` agent.
+    /// Must be valid socket address, of format `IP:PORT` (commonly `127.0.0.1:4040`).
+    #[arg(long)]
+    pub pyroscope_server: Option<String>,
+}
+
+/// An overarching CLI command definition
+#[derive(Debug, clap::Parser)]
+pub struct Cli {
+    /// Possible subcommands with parameters.
+    #[clap(subcommand)]
+    pub subcommand: Option<Subcommand>,
+
+    /// Base command line.
+    #[clap(flatten)]
+    pub run: RunCmd,
+
+    /// Parameters used to create the storage monitor.
+    #[clap(flatten)]
+    pub storage_monitor: sc_storage_monitor::StorageMonitorParams,
+}

cli/src/command.rs

@@ -0,0 +1,369 @@
+use crate::cli::{Cli, Subcommand};
+use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory};
+use futures::future::TryFutureExt;
+use sc_cli::SubstrateCli;
+use service::{
+    self, IdentifyVariant,
+    benchmarking::{
+        benchmark_inherent_data, RemarkBuilder, TransferKeepAliveBuilder
+    },
+};
+use keyring::Sr25519Keyring;
+
+pub use crate::{error::Error, service::BlockId};
+#[cfg(feature = "pyroscope")]
+use pyroscope_pprofrs::{pprof_backend, PprofConfig};
+
+type Result<T> = std::result::Result<T, Error>;
+
+fn get_exec_name() -> Option<String> {
+    std::env::current_exe()
+        .ok()
+        .and_then(|pb| pb.file_name().map(|s| s.to_os_string()))
+        .and_then(|s| s.into_string().ok())
+}
+
+impl SubstrateCli for Cli {
+    fn impl_name() -> String {
+        "Ghost Node".into()
+    }
+
+    fn impl_version() -> String {
+        env!("CARGO_PKG_DESCRIPTION").into()
+    }
+
+    fn description() -> String {
+        env!("CARGO_PKG_DESCRIPTION").into()
+    }
+
+    fn author() -> String {
+        env!("CARGO_PKG_AUTHORS").into()
+    }
+
+    fn support_url() -> String {
+        format!("{}/issues/new", env!("CARGO_PKG_REPOSITORY")).into()
+    }
+
+    fn copyright_start_year() -> i32 {
+        2024
+    }
+
+    fn executable_name() -> String {
+        get_exec_name().unwrap_or("ghost".into())
+    }
+
+    fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+        let id = if id == "" {
+            let n = get_exec_name().unwrap_or_default();
+            ["casper"]
+                .iter()
+                .cloned()
+                .find(|&chain| n.starts_with(chain))
+                .unwrap_or("casper")
+        } else {
+            id
+        };
+
+        Ok(match id {
+            #[cfg(feature = "casper-native")]
+            "casper" => Box::new(service::chain_spec::casper_config()?),
+            #[cfg(feature = "casper-native")]
+            "casper-dev" | "dev" | "development" => Box::new(service::chain_spec::casper_development_config()?),
+            #[cfg(feature = "casper-native")]
+            "casper-local" | "local" => Box::new(service::chain_spec::casper_local_testnet_config()?),
+            #[cfg(feature = "casper-native")]
+            "casper-staging" | "staging" => Box::new(service::chain_spec::casper_staging_testnet_config()?),
+            #[cfg(not(feature = "casper-native"))]
+            name if name.starts_with("casper-") && !name.ends_with(".json") =>
+                Err(format!("`{}` only supported with `casper-native` feature enabled.", name))?,
+            #[cfg(feature = "casper-native")]
+            path => {
+                let path = std::path::PathBuf::from(path);
+
+                let chain_spec = Box::new(
+                   service::GenericChainSpec::from_json_file(path.clone())?
+                ) as Box<dyn service::ChainSpec>;
+
+                if chain_spec.is_casper() {
+                   Box::new(service::CasperChainSpec::from_json_file(path)?)
+                } else {
+                   chain_spec
+                }
+            },
+        })
+    }
+}
+
+fn set_ss58_version(spec: &Box<dyn service::ChainSpec>) {
+    use sp_core::crypto::Ss58AddressFormat;
+
+    let ss58_version = if spec.is_ghost() {
+        Ss58AddressFormat::custom(1995)
+    } else if spec.is_casper() {
+        Ss58AddressFormat::custom(1996)
+    } else {
+        panic!("Invalid network")
+    }
+    .into();
+
+    sp_core::crypto::set_default_ss58_version(ss58_version);
+}
+
+fn run_node_inner<F>(
+    cli: Cli,
+    logger_hook: F,
+) -> Result<()> 
+where
+    F: FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration),
+{
+    let runner = cli
+        .create_runner_with_logger_hook::<sc_cli::RunCmd, F>(&cli.run.base, logger_hook)
+        .map_err(Error::from)?;
+    let chain_spec = &runner.config().chain_spec;
+
+    set_ss58_version(chain_spec);
+
+    runner.run_node_until_exit(move |config| async move {
+        let hwbench = (!cli.run.no_hardware_benchmarks)
+            .then_some(config.database.path().map(|database_path| {
+                let _ = std::fs::create_dir_all(&database_path);
+                sc_sysinfo::gather_hwbench(Some(database_path))
+            })).flatten();
+
+        let database_source = config.database.clone();
+        let task_manager = service::build_full(
+            config,
+            service::NewFullParams {
+                force_authoring_backoff: cli.run.force_authoring_backoff,
+                telemetry_worker_handle: None,
+                hwbench,
+            },
+        ).map(|full| full.task_manager)?;
+
+        if let Some(path) = database_source.path() {
+            sc_storage_monitor::StorageMonitorService::try_spawn(
+                cli.storage_monitor,
+                path.to_path_buf(),
+                &task_manager.spawn_essential_handle(),
+            )?;
+        }
+
+        Ok(task_manager)
+    })
+}
+
+/// Parses ghost specific CLI arguments and run the service.
+pub fn run() -> Result<()> {
+    let cli: Cli = Cli::from_args();
+
+    #[cfg(feature = "pyroscope")]
+    let mut pyroscope_agent_maybe = if let Some(ref agent_addr) = cli.run.pyroscope_server {
+        let address = agent_addr
+            .to_socket_addrs()
+            .map_err(Error::AddressResolutionFailure)?
+            .next()
+            .ok_or_else(|| Error::AddressResolutionMissing)?;
+
+        let agent = pyro::Pyroscope::builder(
+            "http://".to_owned() + address.to_string().as_str(),
+            "ghost".to_owned(),
+        ).backend(pprof_backend(PprofConfig::new().sample_rate(113))).build()?;
+
+        Some(agent.start()?)
+    } else {
+        None
+    };
+
+    #[cfg(not(feature = "pyroscope"))]
+    if cli.run.pyroscope_server.is_some() {
+        return Err(Error::PyroscopeNotCompiledIn)
+    }
+
+    match &cli.subcommand {
+        None => run_node_inner(cli, ghost_metrics::logger_hook()),
+        Some(Subcommand::BuildSpec(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            Ok(runner.sync_run(|config| cmd.run(config.chain_spec, config.network))?)
+        },
+        Some(Subcommand::CheckBlock(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            set_ss58_version(chain_spec);
+
+            runner.async_run(|mut config| {
+                let (client, _, import_queue, task_manager) =
+                    service::new_chain_ops(&mut config)?;
+                Ok((cmd.run(client, import_queue).map_err(Error::SubstrateCli), task_manager))
+            })
+        },
+        Some(Subcommand::ExportBlocks(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            set_ss58_version(chain_spec);
+
+            Ok(runner.async_run(|mut config| {
+                let ( client, _, _, task_manager ) = 
+                    service::new_chain_ops(&mut config)?;
+                Ok((cmd.run(client, config.database).map_err(Error::SubstrateCli), task_manager))
+            })?)
+        },
+        Some(Subcommand::ExportState(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            set_ss58_version(chain_spec);
+
+            Ok(runner.async_run(|mut config| {
+                let ( client, _, _, task_manager ) = 
+                    service::new_chain_ops(&mut config)?;
+                Ok((cmd.run(client, config.chain_spec).map_err(Error::SubstrateCli), task_manager))
+            })?)
+        },
+        Some(Subcommand::ImportBlocks(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            set_ss58_version(chain_spec);
+
+            Ok(runner.async_run(|mut config| {
+                let ( client, _, import_queue, task_manager ) = 
+                    service::new_chain_ops(&mut config)?;
+                Ok((cmd.run(client, import_queue).map_err(Error::SubstrateCli), task_manager))
+            })?)
+        },
+        Some(Subcommand::PurgeChain(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            Ok(runner.sync_run(|config| cmd.run(config.database))?)
+        },
+        Some(Subcommand::Revert(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            set_ss58_version(chain_spec);
+
+            Ok(runner.async_run(|mut config| {
+                let (client, backend, _, task_manager) = 
+                    service::new_chain_ops(&mut config)?;
+
+                let aux_revert = Box::new(|client, backend, blocks| {
+                    service::revert_backend(client, backend, blocks).map_err(|err| {
+                        match err {
+                            service::Error::Blockchain(err) => err.into(),
+                            err => sc_cli::Error::Application(err.into()),
+                        }
+                    })
+                });
+
+                Ok((
+                    cmd.run(client, backend, Some(aux_revert)).map_err(Error::SubstrateCli),
+                    task_manager
+                ))
+            })?)
+        },
+        Some(Subcommand::Benchmark(cmd)) => {
+            let runner = cli.create_runner(cmd).map_err(Error::SubstrateCli)?;
+            let chain_spec = &runner.config().chain_spec;
+
+            match cmd {
+                #[cfg(not(feature = "runtime-benchmarks"))]
+                BenchmarkCmd::Storage(_) => 
+                    return Err(sc_cli::Error::Input(
+                        "Compile with `--feature=runtime-benchmarks \
+                        to enable storage benchmarks.".into()
+                    ).into()),
+                #[cfg(feature = "runtime-benchmarks")]
+                BenchmarkCmd::Storage(cmd) => runner.sync_run(|mut config| {
+                    let (client, backend, _, _,) = 
+                        service::new_chain_ops(&mut config)?;
+                    let db = backend.expose_db();
+                    let storage = backend.expose_storage();
+
+                    cmd.run(config, client.clone(), db, storage).map_err(Error::SubstrateCli)
+                }),
+                BenchmarkCmd::Block(cmd) => runner.sync_run(|mut config| {
+                    let (client, _, _, _,) = 
+                        service::new_chain_ops(&mut config)?;
+                    cmd.run(client.clone()).map_err(Error::SubstrateCli)
+                }),
+                // These commands are very similar and can be handled in nearly the same way
+                BenchmarkCmd::Extrinsic(_) | BenchmarkCmd::Overhead(_) =>
+                    runner.sync_run(|mut config| {
+                        let (client, _, _, _) = 
+                            service::new_chain_ops(&mut config)?;
+
+                        let inherent_data = benchmark_inherent_data()
+                            .map_err(|e| format!("generating inherent data: {:?}", e))?;
+
+                        let remark_builder = RemarkBuilder::new(
+                            client.clone(), 
+                            config.chain_spec.identify_chain(),
+                        );
+
+                        match cmd {
+                            BenchmarkCmd::Extrinsic(cmd) => {
+                                let tka_builder = TransferKeepAliveBuilder::new(
+                                    client.clone(),
+                                    Sr25519Keyring::Alice.to_account_id(),
+                                    config.chain_spec.identify_chain(),
+                                );
+
+                                let ext_factory = ExtrinsicFactory(vec![
+                                    Box::new(remark_builder),
+                                    Box::new(tka_builder),
+                                ]);
+
+                                cmd.run(client.clone(), inherent_data, Vec::new(), &ext_factory)
+                                    .map_err(Error::SubstrateCli)
+                            },
+                            BenchmarkCmd::Overhead(cmd) => cmd.run(
+                                config,
+                                client.clone(),
+                                inherent_data,
+                                Vec::new(),
+                                &remark_builder,
+                            ).map_err(Error::SubstrateCli),
+                            _ => unreachable!("Ensured by the outside match; qed"),
+                        }
+                    }),
+                BenchmarkCmd::Pallet(cmd) => {
+                    set_ss58_version(chain_spec);
+
+                    if cfg!(feature = "runtime-benchmarks") {
+                        runner.sync_run(|config| {
+                            cmd.run_with_spec::<sp_runtime::traits::HashingFor<service::Block>, ()>(
+                                Some(config.chain_spec),
+                            ).map_err(|e| Error::SubstrateCli(e))
+                        })
+                    } else {
+                        Err(sc_cli::Error::Input(
+                            "Benchmarking wasn't enabled when building the node. \
+                            You can enable it with `--features=runtime-benchmarks`.".into()
+                        ).into())
+                    }
+                },
+                BenchmarkCmd::Machine(cmd) => runner.sync_run(|config| {
+                    cmd.run(&config, ghost_machine_primitives::GHOST_NODE_REFERENCE_HARDWARE.clone())
+                        .map_err(Error::SubstrateCli)
+                }),
+                // Note: this allows to implement additional new benchmark
+                // commands.
+                #[allow(unreachable_patterns)]
+                _ => Err(Error::CommandNotImplemented),
+            }
+        },
+        Some(Subcommand::Key(cmd)) => Ok(cmd.run(&cli)?),
+        Some(Subcommand::ChainInfo(cmd)) => {
+            let runner = cli.create_runner(cmd)?;
+            Ok(runner.sync_run(|config| cmd.run::<service::Block>(&config))?)
+        },
+    }?;
+
+    #[cfg(feature = "pyroscope")]
+    if let Some(pyroscope_agent) = pyroscope_agent_maybe.take() {
+        let agent = pyroscope_agent.stop()?;
+        agent.shutdown();
+    }
+    Ok(())
+}

cli/src/error.rs

@@ -0,0 +1,46 @@
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error(transparent)]
+    GhostService(#[from] service::Error),
+
+    #[error(transparent)]
+    SubstrateCli(#[from] sc_cli::Error),
+
+    #[error(transparent)]
+    SubstrateService(#[from] sc_service::Error),
+
+    #[error(transparent)]
+    SubstrateTracing(#[from] sc_tracing::logging::Error),
+
+    #[cfg(not(feature = "pyroscope"))]
+    #[error("Binary was not compiled with `--feature=pyroscope")]
+    PyroscopeNotCompiledIn,
+
+    #[cfg(feature = "pyroscope")]
+    #[error("Failed to connect to pyroscope agent")]
+    PyroscopeError(#[from] pyro::error::PyroscopeError),
+
+    #[error("Failed to resolve provided URL")]
+    AddressResolutionFailure(#[from] std::io::Error),
+
+    #[error("URL did not resolve to anthing")]
+    AddressResolutionMissing,
+
+    #[error("Command not implemented")]
+    CommandNotImplemented,
+
+    #[error(transparent)]
+    Storage(#[from] sc_storage_monitor::Error),
+
+    #[error("Other: {0}")]
+    Other(String),
+
+    #[error("This subcommand is only available when compiled with `{feature}`")]
+    FeatureNotEnabled { feature: &'static str },
+}
+
+impl From<String> for Error {
+    fn from(s: String) -> Self {
+        Self::Other(s)
+    }
+}

cli/src/lib.rs

@@ -0,0 +1,19 @@
+#[cfg(feature = "cli")]
+mod cli;
+#[cfg(feature = "cli")]
+mod command;
+#[cfg(feature = "cli")]
+mod error;
+
+#[cfg(feature = "service")]
+pub use service::{
+    self, Block, CoreApi, IdentifyVariant, ProvideRuntimeApi, TFullClient,
+};
+
+#[cfg(feature = "cli")]
+pub use cli::*;
+#[cfg(feature = "cli")]
+pub use command::*;
+
+#[cfg(feature = "cli")]
+pub use sc_cli::{Error, Result};