//+------------------------------------------------------------------+
//|  PFX_Journal.cs  -  log NinjaTrader 8 fills to PrecisionFX        |
//|  (c) 2026 Daniel Freeman - PrecisionFX.io                         |
//+------------------------------------------------------------------+
//  1.02 - 14 Oct 2026. Optional  offset=50000  after a link in journal.txt. Some prop firms' funded accounts start
//         at $0 in NinjaTrader (MFF Sim Funded) while the firm's account is 50,000: the offset is added to the balance
//         and equity that are sent, so the journal's balance, growth % and prop rules agree. Never touches fills.
//  1.01 - 14 Oct 2026. Also sends the account's balance, net liquidation, open P&L, margin and open-position count
//         once a minute (the NT8 twin of MT5 Journal 1.03), so the account card shows real equity, not "-".
//         Needs the website update that accepts a "snapshot" (deployed 14 Oct). Against an older website the
//         balance send is refused, logged once, and retried every 30 minutes; fills are never affected.
//  1.00a - 14 Oct 2026. Messages also go to PFX\journal.log (a busy Output window hides them).
//  1.00 - 26 Sep 2026. First build. NOT YET COMPILED (NT8 isn't on the
//         build machine) - Dan: F5 in the NinjaScript Editor, then copy back.
//
//  What it does
//    Every fill on the accounts you choose is sent to your PrecisionFX
//    journal, so NT8 trades log themselves - the NT8 twin of PFX Journal for
//    MT5. It NEVER trades: no order is created, changed or cancelled here.
//
//  Set-up
//    1. On app.precisionfx.io add an account of type "NinjaTrader 8" and copy
//       its private link.
//    2. Create  Documents\NinjaTrader 8\PFX\journal.txt  with one line per
//       NT8 account:     Sim101 = https://app.precisionfx.io/api/ingest/nt8/pfx_...
//       (lines starting with # are ignored; the account name is as NT8 shows it)
//    3. Put this file in  Documents\NinjaTrader 8\bin\Custom\AddOns\ , open the
//       NinjaScript Editor and press F5. It starts by itself; the file is
//       re-read every minute, so a new line needs no restart.
//
//  What gets sent (the contract: PrecisionFX-Website/src/server/journal/ingest-nt8.ts)
//    id          NT8's ExecutionId  - the server ignores any it already has,
//                                     so a retry or a re-send changes nothing
//    instrument  "MGC 12-26"
//    side        buy / sell
//    qty, price
//    commission  NT8's own figure, a positive cost
//    pointValue  the contract's point value (MGC 10, MNQ 2) - the journal
//                needs it to turn a price move into money
//    timeMs      UNIX ms in UTC, converted from NT8's own timezone setting
//
//  On start it also sends the fills NT8 already holds for today, so anything
//  from before NT8 opened this session is picked up. Older history: use the
//  journal's CSV import (Trade Performance -> Trades -> Export).
//
//  Safety
//    - Nothing in here can throw into NinjaTrader: every failure is logged to
//      the Output window (tab 1) and the fill is retried.
//    - Web calls run on a background timer, never on NinjaTrader's threads.
//    - Unsent fills are kept in memory and retried every 5 s (capped at 5,000);
//      if NT8 closes first, today's fills are sent again on the next start.
//    - The private link is never printed in full.
//+------------------------------------------------------------------+
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NinjaTrader.Cbi;
using NinjaTrader.Code;

namespace NinjaTrader.NinjaScript.AddOns
{
	public class PfxJournal : AddOnBase
	{
		public const string Version = "1.02";
		const int MaxQueued = 5000;
		const int BatchSize = 200;

		static readonly HttpClient http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
		static readonly CultureInfo Inv = CultureInfo.InvariantCulture;

		sealed class Fill
		{
			public string Account, Id, Json;
		}

		readonly ConcurrentQueue<Fill> queue = new ConcurrentQueue<Fill>();
		readonly ConcurrentDictionary<string, byte> seen = new ConcurrentDictionary<string, byte>();
		readonly HashSet<Account> subscribed = new HashSet<Account>();
		readonly object subLock = new object();
		Dictionary<string, string> links = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
		Dictionary<string, double> offsets = new Dictionary<string, double>(StringComparer.OrdinalIgnoreCase);
		DateTime configReadAt = DateTime.MinValue;
		readonly Dictionary<string, DateTime> snapDue = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
		readonly HashSet<string> snapOk = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
		System.Threading.Timer timer;
		int flushing;

		static string ConfigPath
		{
			get { return Path.Combine(NinjaTrader.Core.Globals.UserDataDir, "PFX", "journal.txt"); }
		}

		protected override void OnStateChange()
		{
			try
			{
				if (State == State.SetDefaults)
				{
					Name = "PFX Journal";
					Description = "Sends NinjaTrader fills to your PrecisionFX journal. Never trades.";
				}
				else if (State == State.Active)
				{
					Log("v" + Version + " starting - config: " + ConfigPath);
					ReadConfig();
					SubscribeConfigured();
					timer = new System.Threading.Timer(_ => Tick(), null, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5));
				}
				else if (State == State.Terminated)
				{
					if (timer != null) { timer.Dispose(); timer = null; }
					lock (subLock)
					{
						foreach (Account a in subscribed) a.ExecutionUpdate -= OnExecution;
						subscribed.Clear();
					}
				}
			}
			catch (Exception ex) { Log("state " + State + " failed - " + ex.GetType().Name + ": " + ex.Message); }
		}

		// ── config ──────────────────────────────────────────────────────────

		void ReadConfig()
		{
			configReadAt = DateTime.UtcNow;
			var next = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
			var nextOffsets = new Dictionary<string, double>(StringComparer.OrdinalIgnoreCase);
			try
			{
				if (!File.Exists(ConfigPath))
				{
					if (links.Count == 0) Log("no " + ConfigPath + " yet - made a template; put a line in it like:  Sim101 = <your private link>");
					try
					{
						Directory.CreateDirectory(Path.GetDirectoryName(ConfigPath));
						File.WriteAllText(ConfigPath,
							"# PFX Journal - one line per NinjaTrader account:  <account name exactly as NinjaTrader shows it> = <private link>" + Environment.NewLine +
							"# Get the link on app.precisionfx.io -> Accounts -> your NinjaTrader account. Example:" + Environment.NewLine +
							"# Sim101 = https://app.precisionfx.io/api/ingest/nt8/pfx_xxxxxxxxxxxxxxxxxxxxxxxx" + Environment.NewLine +
							"# A funded prop account that starts at 0 in NinjaTrader can add its size:  ... pfx_xxxx offset=50000" + Environment.NewLine);
					}
					catch { }
					links = next;
					return;
				}
				foreach (string raw in File.ReadAllLines(ConfigPath))
				{
					string line = raw.Trim();
					if (line.Length == 0 || line.StartsWith("#")) continue;
					int eq = line.IndexOf('=');
					if (eq <= 0) { Log("skipping a line without '=': " + Mask(line)); continue; }
					string name = line.Substring(0, eq).Trim();
					string[] parts = line.Substring(eq + 1).Trim().Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
					string url = parts.Length > 0 ? parts[0] : "";
					double off = 0;
					bool badOption = false;
					for (int i = 1; i < parts.Length; i++)
					{
						string o = parts[i];
						double v;
						if (o.StartsWith("offset=", StringComparison.OrdinalIgnoreCase) && double.TryParse(o.Substring(7), NumberStyles.Float, Inv, out v) && Finite(v) && Math.Abs(v) < 1e9) off = v;
						else { Log("skipping " + name + " - couldn't read '" + o + "' (the only option is offset=50000)"); badOption = true; break; }
					}
					if (badOption) continue;
					if (!url.StartsWith("https://") || url.IndexOf("/api/ingest/nt8/pfx_", StringComparison.Ordinal) < 0)
					{
						Log("skipping " + name + " - that isn't an NT8 private link from the journal's Accounts page");
						continue;
					}
					next[name] = url;
					if (off != 0) nextOffsets[name] = off;
				}
			}
			catch (Exception ex) { Log("couldn't read " + ConfigPath + " - " + ex.Message); return; }
			if (!SameLinks(links, next)) Log(next.Count + " account(s) linked: " + string.Join(", ", next.Keys.Select(k => k + (nextOffsets.ContainsKey(k) ? " (offset " + Num(nextOffsets[k]) + ")" : ""))));
			else if (nextOffsets.Count != offsets.Count || nextOffsets.Any(kv => !offsets.ContainsKey(kv.Key) || offsets[kv.Key] != kv.Value)) Log("offsets: " + (nextOffsets.Count == 0 ? "none" : string.Join(", ", nextOffsets.Select(kv => kv.Key + " " + Num(kv.Value)))));
			links = next;
			offsets = nextOffsets;
		}

		static bool SameLinks(Dictionary<string, string> a, Dictionary<string, string> b)
		{
			return a.Count == b.Count && a.All(kv => b.ContainsKey(kv.Key) && b[kv.Key] == kv.Value);
		}

		/** Subscribe to every connected account that has a link; send what it already holds for today. */
		void SubscribeConfigured()
		{
			List<Account> accounts;
			lock (Account.All) accounts = Account.All.ToList();
			foreach (Account a in accounts)
			{
				if (!links.ContainsKey(a.Name)) continue;
				bool added;
				lock (subLock) added = subscribed.Add(a);
				if (!added) continue;
				a.ExecutionUpdate += OnExecution;
				int n = 0;
				lock (a.Executions)
				{
					foreach (Execution e in a.Executions) { if (Enqueue(e)) n++; }
				}
				Log("watching " + a.Name + (n > 0 ? " - " + n + " fill(s) from today queued" : ""));
			}
		}

		// ── fills ───────────────────────────────────────────────────────────

		void OnExecution(object sender, ExecutionEventArgs e)
		{
			try { if (e != null && e.Execution != null) Enqueue(e.Execution); }
			catch (Exception ex) { Log("fill not queued - " + ex.GetType().Name + ": " + ex.Message); }
		}

		bool Enqueue(Execution e)
		{
			if (e == null || e.Account == null || e.Instrument == null || string.IsNullOrEmpty(e.ExecutionId)) return false;
			if (!seen.TryAdd(e.Account.Name + "|" + e.ExecutionId, 0)) return false; // already queued this session
			if (queue.Count >= MaxQueued) { Log("queue full (" + MaxQueued + ") - is the journal reachable? oldest fills will come back on next start"); return false; }

			string side = e.MarketPosition == MarketPosition.Long ? "buy" : "sell";
			double pv = e.Instrument.MasterInstrument.PointValue;
			DateTime utc = ToUtc(e.Time);
			long ms = (long)(utc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;

			var sb = new StringBuilder(220);
			sb.Append("{\"id\":").Append(Q(e.ExecutionId));
			sb.Append(",\"instrument\":").Append(Q(e.Instrument.FullName));
			sb.Append(",\"side\":\"").Append(side).Append('"');
			sb.Append(",\"qty\":\"").Append(e.Quantity.ToString(Inv)).Append('"');
			sb.Append(",\"price\":\"").Append(Num(e.Price)).Append('"');
			sb.Append(",\"commission\":\"").Append(Num(Math.Abs(e.Commission))).Append('"');
			sb.Append(",\"pointValue\":\"").Append(Num(pv)).Append('"');
			sb.Append(",\"timeMs\":").Append(ms.ToString(Inv)).Append('}');

			queue.Enqueue(new Fill { Account = e.Account.Name, Id = e.ExecutionId, Json = sb.ToString() });
			return true;
		}

		/** NT8 reports times in its own configured timezone (Tools > Options > General). */
		static DateTime ToUtc(DateTime t)
		{
			try
			{
				TimeZoneInfo tz = NinjaTrader.Core.Globals.GeneralOptions.TimeZoneInfo ?? TimeZoneInfo.Local;
				return TimeZoneInfo.ConvertTimeToUtc(DateTime.SpecifyKind(t, DateTimeKind.Unspecified), tz);
			}
			catch { return t.ToUniversalTime(); }
		}

		// ── sending ─────────────────────────────────────────────────────────

		void Tick()
		{
			try
			{
				if ((DateTime.UtcNow - configReadAt).TotalSeconds >= 60) ReadConfig();
				SubscribeConfigured(); // picks up accounts that connect later
				if (Interlocked.Exchange(ref flushing, 1) == 1) return; // previous flush still running
				Task.Run(async () =>
				{
					try { await Flush(); await SendSnapshots(); }
					catch (Exception ex) { Log("send failed - " + ex.GetType().Name + ": " + ex.Message); }
					finally { Interlocked.Exchange(ref flushing, 0); }
				});
			}
			catch (Exception ex) { Log("tick failed - " + ex.GetType().Name + ": " + ex.Message); }
		}

		async Task Flush()
		{
			if (queue.IsEmpty) return;
			var batch = new List<Fill>();
			Fill f;
			while (batch.Count < MaxQueued && queue.TryDequeue(out f)) batch.Add(f);

			var retry = new List<Fill>();
			foreach (var group in batch.GroupBy(x => x.Account))
			{
				string url;
				if (!links.TryGetValue(group.Key, out url)) { retry.AddRange(group); continue; } // link removed: hold
				foreach (var chunk in group.Select((x, i) => new { x, i }).GroupBy(p => p.i / BatchSize, p => p.x))
				{
					List<Fill> part = chunk.ToList();
					string body = "{\"fills\":[" + string.Join(",", part.Select(p => p.Json)) + "]}";
					try
					{
						using (var content = new StringContent(body, Encoding.UTF8, "application/json"))
						using (HttpResponseMessage res = await http.PostAsync(url, content).ConfigureAwait(false))
						{
							int code = (int)res.StatusCode;
							if (code == 200)
							{
								string text = await res.Content.ReadAsStringAsync().ConfigureAwait(false);
								Log(group.Key + ": sent " + part.Count + " fill(s) - " + Short(text));
							}
							else if (code == 401 || code == 403 || code == 422)
							{
								// the journal refused them: retrying the same thing won't help
								string text = await res.Content.ReadAsStringAsync().ConfigureAwait(false);
								Log(group.Key + ": journal said " + code + " - " + Short(text) + (code == 401 ? " (check the link in journal.txt)" : ""));
							}
							else
							{
								Log(group.Key + ": journal said " + code + " - will retry");
								retry.AddRange(part);
							}
						}
					}
					catch (Exception ex)
					{
						Log(group.Key + ": can't reach the journal (" + ex.GetType().Name + ") - will retry");
						retry.AddRange(part);
					}
				}
			}
			foreach (Fill r in retry) queue.Enqueue(r);
		}

		// ── balance snapshots (1.01) ────────────────────────────────────────

		/** Once a minute per linked, connected account: what the account is worth right now. */
		async Task SendSnapshots()
		{
			List<Account> accounts;
			lock (Account.All) accounts = Account.All.ToList();
			foreach (Account a in accounts)
			{
				string url;
				if (!links.TryGetValue(a.Name, out url)) continue;
				if (a.ConnectionStatus != ConnectionStatus.Connected) continue;
				DateTime due;
				if (snapDue.TryGetValue(a.Name, out due) && DateTime.UtcNow < due) continue;

				double offset;
				offsets.TryGetValue(a.Name, out offset);
				string body = SnapshotJson(a, offset);
				if (body == null) continue;   // account data not loaded yet: try again next tick
				snapDue[a.Name] = DateTime.UtcNow.AddSeconds(60);
				try
				{
					using (var content = new StringContent(body, Encoding.UTF8, "application/json"))
					using (HttpResponseMessage res = await http.PostAsync(url, content).ConfigureAwait(false))
					{
						int code = (int)res.StatusCode;
						if (code == 200)
						{
							if (snapOk.Add(a.Name)) Log(a.Name + ": balance and equity are being sent every minute");
						}
						else if (code == 400 || code == 404 || code == 422)
						{
							// an older website does not know snapshots yet: say so once, ask again in 30 minutes
							snapDue[a.Name] = DateTime.UtcNow.AddMinutes(30);
							snapOk.Remove(a.Name);
							Log(a.Name + ": journal said " + code + " to the balance - the website may need its update; trying again in 30 min");
						}
						else if (code == 401 || code == 403)
						{
							snapDue[a.Name] = DateTime.UtcNow.AddMinutes(30);
							Log(a.Name + ": journal said " + code + " to the balance - check the link in journal.txt");
						}
						else Log(a.Name + ": journal said " + code + " to the balance - will retry");
					}
				}
				catch (Exception ex) { Log(a.Name + ": can't send the balance (" + ex.GetType().Name + ") - will retry"); }
			}
		}

		/** null when the account's numbers are not usable yet (not loaded, NaN, or a zero balance AND zero equity). */
		static string SnapshotJson(Account a, double offset)
		{
			try
			{
				Currency cur = a.Denomination;
				double balance = a.Get(AccountItem.CashValue, cur);
				double equity = a.Get(AccountItem.NetLiquidation, cur);
				double floating = a.Get(AccountItem.UnrealizedProfitLoss, cur);
				double margin = a.Get(AccountItem.InitialMargin, cur);
				if (!Finite(balance) || !Finite(equity) || !Finite(floating) || !Finite(margin)) return null;
				if (balance == 0 && equity == 0 && offset == 0) return null;
				balance += offset;   // a funded account that starts at $0 in NinjaTrader but is 50,000 at the firm
				equity += offset;

				int open = 0;
				lock (a.Positions) { foreach (Position p in a.Positions) if (p.MarketPosition != MarketPosition.Flat) open++; }
				long secs = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds;

				var sb = new StringBuilder(200);
				sb.Append("{\"snapshot\":{\"time\":").Append(secs.ToString(Inv));
				sb.Append(",\"balance\":\"").Append(Num(balance)).Append('"');
				sb.Append(",\"equity\":\"").Append(Num(equity)).Append('"');
				sb.Append(",\"floating\":\"").Append(Num(floating)).Append('"');
				sb.Append(",\"margin\":\"").Append(Num(Math.Abs(margin))).Append('"');
				sb.Append(",\"openPositions\":").Append(open.ToString(Inv));
				string code = CurrencyCode(cur);
				if (code != null) sb.Append(",\"currency\":\"").Append(code).Append('"');
				return sb.Append("}}").ToString();
			}
			catch { return null; }
		}

		static bool Finite(double v) { return !double.IsNaN(v) && !double.IsInfinity(v); }

		static string CurrencyCode(Currency c)
		{
			switch (c)
			{
				case Currency.UsDollar: return "USD";
				case Currency.BritishPound: return "GBP";
				case Currency.Euro: return "EUR";
				default: return null;
			}
		}

		// ── helpers ─────────────────────────────────────────────────────────

		static string Num(double v) { return v.ToString("0.##########", Inv); }

		static string Q(string s)
		{
			var sb = new StringBuilder(s.Length + 2).Append('"');
			foreach (char c in s)
			{
				if (c == '"' || c == '\\') sb.Append('\\').Append(c);
				else if (c < 32) sb.Append("\\u").Append(((int)c).ToString("x4"));
				else sb.Append(c);
			}
			return sb.Append('"').ToString();
		}

		static string Short(string s) { return s == null ? "" : (s.Length > 160 ? s.Substring(0, 160) + "..." : s); }

		/** Never print a private link in full. */
		static string Mask(string s)
		{
			int i = s.IndexOf("pfx_", StringComparison.Ordinal);
			return i < 0 ? s : s.Substring(0, Math.Min(s.Length, i + 8)) + "...";
		}

		static readonly object logLock = new object();

		/** Output window (tab 1) AND PFX\journal.log, so a busy Output window (a backtest) can't hide the messages. */
		static void Log(string s)
		{
			try { Output.Process("PFX Journal - " + s, PrintTo.OutputTab1); } catch { }
			try
			{
				string path = Path.Combine(NinjaTrader.Core.Globals.UserDataDir, "PFX", "journal.log");
				lock (logLock)
				{
					if (File.Exists(path) && new FileInfo(path).Length > 500000) File.Delete(path);
					File.AppendAllText(path, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture) + "  " + s + Environment.NewLine);
				}
			}
			catch { }
		}
	}
}