Skip to content

GitLab

Unverified Commit 75028bd7 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #235 from microsoft/master 

merge master
parents 1d74ae5e 2e42d1d8
Hide whitespace changes
Inline Side-by-side
Showing with 13066 additions and 2 deletions
src/nasui/src/App.css 0 → 100644
View file @ 75028bd7
svg {
overflow: hidden;
}
.node {
white-space: nowrap;
}
.node.input rect, .node.output rect {
fill: #9d0f0f;
}
.node.hub rect {
fill: #0f309d;
}
.node.blob rect {
fill: #0F9D58;
}
.node text {
fill: #fff;
font-family: "Roboto", "Helvetica", "Arial", sans-serif;
font-weight: 500;
}
.cluster rect {
stroke: #333;
fill: #000;
fill-opacity: 0.1;
stroke-width: 1.5px;
}
.edgePath path.path {
stroke: #333;
stroke-width: 1.5px;
fill: none;
}
src/nasui/src/App.tsx 0 → 100644
View file @ 75028bd7
import React, { ChangeEvent } from 'react';
import './App.css';
import 'typeface-roboto';
import { createStyles, withStyles } from '@material-ui/core/styles';
import AppBar from '@material-ui/core/AppBar';
import Button from '@material-ui/core/Button';
import Grid from '@material-ui/core/Grid';
import IconButton from '@material-ui/core/IconButton';
import Slider from '@material-ui/core/Slider';
import Toolbar from '@material-ui/core/Toolbar';
import Typography from '@material-ui/core/Typography';
import RefreshIcon from '@material-ui/icons/Refresh';
import SettingsIcon from '@material-ui/icons/Settings';
import CloseIcon from '@material-ui/icons/Close';
import ShuffleIcon from '@material-ui/icons/Shuffle';
import Snackbar from '@material-ui/core/Snackbar';
import FormControl from '@material-ui/core/FormControl';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import FormGroup from '@material-ui/core/FormGroup';
import Checkbox from '@material-ui/core/Checkbox';
import Dialog from '@material-ui/core/Dialog';
import DialogTitle from '@material-ui/core/DialogTitle';
import DialogContent from '@material-ui/core/DialogContent';
import DialogActions from '@material-ui/core/DialogActions';
import MuiExpansionPanel from '@material-ui/core/ExpansionPanel';
import MuiExpansionPanelSummary from '@material-ui/core/ExpansionPanelSummary';
import MuiExpansionPanelDetails from '@material-ui/core/ExpansionPanelDetails';
import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
import List from '@material-ui/core/List';
import ListItem from '@material-ui/core/ListItem';
import Backdrop from '@material-ui/core/Backdrop';
import Chart from './Chart';
import { Graph } from './graphUtils';
const styles = createStyles({
bottomAppBar: {
top: 'auto',
bottom: 0,
zIndex: 'auto',
},
title: {
flexGrow: 1,
textAlign: 'left'
},
panel: {
position: 'absolute',
top: 76,
right: 16,
width: 400,
},
listItem: {
paddingLeft: 0,
paddingRight: 0,
paddingTop: 2,
paddingBottom: 2,
fontSize: '0.8em',
wordBreak: 'break-all',
},
listSubtitle: {
fontWeight: 600,
paddingLeft: 0,
paddingRight: 0,
fontSize: '0.9em',
},
listTitle: {
lineHeight: 1.1,
wordBreak: 'break-all'
},
backdrop: {
color: '#fff',
zIndex: 100,
},
snackbar: {
bottom: 76
}
});
const ExpansionPanel = withStyles({
root: {
'&$expanded': {
margin: 'auto',
},
},
expanded: {},
})(MuiExpansionPanel);
const ExpansionPanelSummary = withStyles({
root: {},
content: {
'&$expanded': {
margin: '12px 0',
},
},
expanded: {},
})(MuiExpansionPanelSummary);
const ExpansionPanelDetails = withStyles(theme => ({
root: {
paddingTop: 0,
paddingBottom: theme.spacing(1),
},
}))(MuiExpansionPanelDetails);
type AppState = {
graph: Graph | undefined,
graphData: any,
logData: any[],
sliderValue: number,
maxSliderValue: number,
sliderStep: number,
settingsOpen: boolean,
hideSidechainNodes: boolean,
hidePrimitiveNodes: boolean,
snackbarOpen: boolean,
selectedNode: string,
loading: boolean,
layout: boolean,
}
type AppProps = {
classes: any
}
class App extends React.Component<AppProps, AppState> {
constructor(props: any) {
super(props);
this.state = {
graph: undefined,
graphData: undefined,
logData: [],
sliderValue: 0,
maxSliderValue: 0,
sliderStep: 1,
settingsOpen: false,
hideSidechainNodes: true,
hidePrimitiveNodes: true,
selectedNode: '',
loading: false,
snackbarOpen: false,
layout: false,
};
this.refresh = this.refresh.bind(this);
}
componentDidMount() {
this.refresh();
}
refresh() {
this.setState({ loading: true });
fetch('/refresh')
.then((response) => { return response.json() })
.then((data) => {
const graph = new Graph(data.graph, this.state.hideSidechainNodes);
this.setState({
graphData: data.graph,
graph: graph,
logData: data.log,
maxSliderValue: data.log.length - 1,
sliderStep: Math.max(1, Math.floor(data.log.length / 20)),
sliderValue: Math.min(data.log.length, this.state.sliderValue),
loading: false,
snackbarOpen: graph.nodes.length > 100
});
});
}
private renderExpansionPanel() {
const { classes } = this.props;
const { selectedNode, graph } = this.state;
if (graph === undefined)
return null;
const info = graph.nodeSummary(selectedNode);
if (info === undefined)
return null;
const subtitle = info.op ?
(info.op === 'IO Node' ? info.op : `Operation: ${info.op}`) :
`Subgraph: ${info.nodeCount} nodes, ${info.edgeCount} edges`;
return (
<ExpansionPanel className={classes.panel}>
<ExpansionPanelSummary
expandIcon={<ExpandMoreIcon />}
>
<Typography variant='subtitle1' className={classes.listTitle}><b>{info.name}</b><br />{subtitle}</Typography>
</ExpansionPanelSummary>
<ExpansionPanelDetails>
<List dense={true} style={{
maxHeight: window.innerHeight * .5,
overflowY: 'auto',
paddingTop: 0,
width: '100%'
}}>
{
info.attributes &&
<React.Fragment>
<ListItem className={classes.listSubtitle}>Attributes</ListItem>
<ListItem className={classes.listItem}>{info.attributes}</ListItem>
</React.Fragment>
}
{
info.inputs.length > 0 &&
<React.Fragment>
<ListItem className={classes.listSubtitle}>Inputs ({info.inputs.length})</ListItem>
{
info.inputs.map((item, i) => <ListItem className={classes.listItem} key={`input${i}`}>{item}</ListItem>)
}
</React.Fragment>
}
{
info.outputs.length > 0 &&
<React.Fragment>
<ListItem className={classes.listSubtitle}>Outputs ({info.outputs.length})</ListItem>
{
info.outputs.map((item, i) => <ListItem className={classes.listItem} key={`output${i}`}>{item}</ListItem>)
}
</React.Fragment>
}
</List>
</ExpansionPanelDetails>
</ExpansionPanel>
);
}
render() {
const { classes } = this.props;
const { sliderValue, maxSliderValue, sliderStep, settingsOpen, loading, snackbarOpen } = this.state;
const handleSliderChange = (event: ChangeEvent<{}>, value: number | number[]) => {
this.setState({ sliderValue: value as number });
};
const handleSettingsDialogToggle = (value: boolean) => () => {
this.setState({ settingsOpen: value });
};
const handleSettingsChange = (name: string) => (event: React.ChangeEvent<HTMLInputElement>) => {
this.setState({
...this.state,
[name]: event.target.checked
}, () => {
this.setState({
graph: new Graph(this.state.graphData, this.state.hideSidechainNodes),
})
});
};
const handleSelectionChange = (node: string) => {
this.setState({
selectedNode: node
});
};
const handleLoadingState = (state: boolean) => () => {
this.setState({ loading: state });
};
const handleSnackbarClose = () => {
this.setState({ snackbarOpen: false });
};
const handleLayoutStateChanged = (state: boolean) => () => {
this.setState({ layout: state });
};
return (
<div className='App'>
<Chart
width={window.innerWidth}
height={window.innerHeight}
graph={this.state.graph}
activation={sliderValue < this.state.logData.length ? this.state.logData[sliderValue] : undefined}
handleSelectionChange={handleSelectionChange}
onRefresh={handleLoadingState(true)}
onRefreshComplete={handleLoadingState(false)}
layout={this.state.layout}
onLayoutComplete={handleLayoutStateChanged(false)}
/>
<AppBar position='fixed' color='primary'>
<Toolbar>
<Typography variant='h6' className={classes.title}>
NNI NAS Board
</Typography>
<IconButton color='inherit' onClick={handleLayoutStateChanged(true)}>
<ShuffleIcon />
</IconButton>
<IconButton color='inherit' onClick={this.refresh}>
<RefreshIcon />
</IconButton>
<IconButton color='inherit' onClick={handleSettingsDialogToggle(true)}>
<SettingsIcon />
</IconButton>
</Toolbar>
</AppBar>
<AppBar position='fixed' color='default' className={classes.bottomAppBar}>
<Toolbar variant='dense'>
<Grid container spacing={2} alignItems='center'>
<Grid item xs>
<Slider
value={sliderValue}
max={maxSliderValue}
min={0}
step={sliderStep}
onChange={handleSliderChange}
/>
</Grid>
<Grid item>
<Typography variant='body1'>
{sliderValue}/{maxSliderValue}
</Typography>
</Grid>
</Grid>
</Toolbar>
</AppBar>
<Dialog onClose={handleSettingsDialogToggle(false)} open={settingsOpen}>
<DialogTitle>Settings</DialogTitle>
<DialogContent>
<FormControl component='fieldset'>
<FormGroup>
<FormControlLabel
control={<Checkbox checked={this.state.hideSidechainNodes}
onChange={handleSettingsChange('hideSidechainNodes')}
value='hideSidechainNodes' />}
label='Hide sidechain nodes'
/>
{ // TODO: hide primitive nodes
/* <FormControlLabel
control={<Checkbox checked={this.state.hidePrimitiveNodes}
onChange={handleSettingsChange('hidePrimitiveNodes')}
value='hidePrimitiveNodes' />}
label='Hide primitive nodes'
/> */}
</FormGroup>
</FormControl>
</DialogContent>
<DialogActions>
<Button onClick={handleSettingsDialogToggle(false)} color='primary'>
Close
</Button>
</DialogActions>
</Dialog>
{this.renderExpansionPanel()}
<Snackbar
className={classes.snackbar}
anchorOrigin={{
vertical: 'bottom',
horizontal: 'left',
}}
open={snackbarOpen}
message='Graph is too large. Might induce performance issue.'
onClose={handleSnackbarClose}
action={
<IconButton size='small' color='inherit' onClick={handleSnackbarClose}>
<CloseIcon fontSize='small' />
</IconButton>
}
/>
{
loading && <Backdrop className={classes.backdrop} open={true}>
<Typography>Loading...</Typography>
</Backdrop>
}
</div>
);
}
}
export default withStyles(styles)(App);
src/nasui/src/Chart.tsx 0 → 100644
View file @ 75028bd7
import React, { createRef } from 'react';
import cytoscape from 'cytoscape';
import dagre from 'cytoscape-dagre';
import lodash from 'lodash';
import { Graph, NodeTs } from './graphUtils';
cytoscape.use(dagre);
type ChartProps = {
width: number,
height: number,
graph: Graph | undefined,
activation: any,
handleSelectionChange: (_: string) => void,
onRefresh: () => void,
onRefreshComplete: () => void,
layout: boolean,
onLayoutComplete: () => void,
}
const styles = [
{
selector: ':active',
style: {
'overlay-opacity': 0.1,
}
},
{
selector: 'node',
style: {
'shape': 'round-rectangle',
'width': 'label',
'height': 'label',
'content': 'data(label)',
'text-wrap': 'wrap',
'text-valign': 'center',
'text-halign': 'center',
'font-family': '"Roboto", "Helvetica", "Arial", sans-serif',
'font-size': 20,
'padding-left': '8px',
'padding-right': '8px',
'padding-top': '8px',
'padding-bottom': '8px',
'background-color': '#000',
'background-opacity': 0.02,
'border-color': '#555',
'border-width': '2px',
}
},
{
selector: 'node.op',
style: {
'background-color': '#fafafa',
'background-opacity': 1,
}
},
{
selector: 'node:selected',
style: {
'border-width': '4px',
'border-color': '#101010',
}
},
{
selector: 'edge',
style: {
'curve-style': 'bezier',
'target-arrow-shape': 'triangle',
'line-color': '#555',
'target-arrow-color': '#555'
}
},
{
selector: 'node.compound',
style: {
'shape': 'roundrectangle',
'text-valign': 'top',
'padding-top': '30px',
'padding-bottom': '10px',
'padding-right': '10px',
'padding-left': '10px',
}
}
];
export default class Chart extends React.Component<ChartProps> {
container = createRef<HTMLDivElement>();
zoom: any;
collapseInstance: any;
cyInstance: cytoscape.Core | null = null;
expandSet: Set<string> = new Set<string>();
elemWeight: Map<string, number> = new Map<string, number>();
graphEl: any[] = [];
componentDidMount() {
this.cyInstance = cytoscape({
container: this.container.current,
elements: [],
style: styles as any,
wheelSensitivity: 0.1,
});
let singleClickedNodes = new Set<string>();
this.cyInstance!.off('click');
this.cyInstance!.on('click', (e: cytoscape.EventObject) => {
if (e.target.id) {
let nodeId = e.target.id();
if (singleClickedNodes.has(nodeId)) {
singleClickedNodes.delete(nodeId);
e.target.trigger('dblclick');
} else {
singleClickedNodes.add(nodeId);
setTimeout(() => { singleClickedNodes.delete(nodeId); }, 300);
}
}
});
this.cyInstance!.on('dblclick', (e: cytoscape.EventObject) => {
const nodeId = e.target.id();
const node = this.props.graph!.getNodeById(nodeId);
if (node !== undefined && node.isParent()) {
if (this.expandSet.has(nodeId)) {
this.expandSet.delete(nodeId);
} else {
this.expandSet.add(nodeId);
}
this.renderGraph(false);
}
});
this.cyInstance!.on('select', (e: cytoscape.EventObject) => {
this.props.handleSelectionChange(e.target.id());
});
this.renderGraph(true);
}
componentDidUpdate(prevProps: ChartProps) {
if (prevProps.graph === this.props.graph) {
if (prevProps.width === this.props.width &&
prevProps.height === this.props.height) {
if (this.props.layout) {
this.props.onLayoutComplete();
this.reLayout();
}
if (prevProps.activation !== this.props.activation) {
// perhaps only display step is changed
this.applyMutableWeight();
}
} else {
// something changed, re-render
this.renderGraph(false);
}
} else {
// re-calculate collapse
this.renderGraph(true);
}
}
private graphElements() {
let graphElements: any[] = [];
const { graph } = this.props;
if (graph === undefined)
return [];
const collapseMap = new Map<string, string>();
const traverse = (node: NodeTs, top: NodeTs | undefined) => {
collapseMap.set(node.id, top === undefined ? node.id : top.id);
if (node.id && (top === undefined || node === top)) {
// not root and will display
const isCompound = node.isParent() && this.expandSet.has(node.id);
let data: any = {
id: node.id,
label: node.op ? node.op : node.tail,
};
if (node.parent !== undefined)
data.parent = node.parent.id;
const classes = [];
if (isCompound) classes.push('compound');
if (node.op) classes.push('op');
graphElements.push({
data: data,
classes: classes
});
}
for (const child of node.children)
traverse(child, node.id && top === undefined && !this.expandSet.has(node.id) ? node : top);
}
traverse(graph.root, undefined);
graph.edges.forEach(edge => {
const [srcCollapse, trgCollapse] = [edge.source, edge.target].map((node) => collapseMap.get(node.id)!);
if (edge.source !== edge.target && srcCollapse === trgCollapse) {
return;
}
graphElements.push({
data: {
id: edge.id,
source: srcCollapse,
target: trgCollapse
}
});
});
return graphElements;
}
private applyMutableWeight() {
const { graph, activation } = this.props;
if (graph === undefined || activation === undefined)
return;
const weights = graph.weightFromMutables(activation);
weights.forEach((weight, elem) => {
if (this.elemWeight.get(elem) !== weight) {
this.cyInstance!.getElementById(elem).style({
opacity: 0.2 + 0.8 * weight
});
}
});
this.elemWeight = weights;
}
private graphElDifference(prev: any[], next: any[]): [Set<string>, any] {
const tracedElements = new Set(prev.map(ele => ele.data.id));
const prevMap = new Map(prev.map(ele => [ele.data.id, ele]));
const nextMap = new Map(next.map(ele => [ele.data.id, ele]));
const addedEles: any = [];
nextMap.forEach((val, k) => {
const prevEle = prevMap.get(k);
if (prevEle === undefined) {
addedEles.push(val);
} else if (!lodash.isEqual(val, prevEle)) {
tracedElements.delete(k);
addedEles.push(val);
} else {
tracedElements.delete(k);
}
});
return [tracedElements, addedEles];
}
private reLayout() {
this.props.onRefresh();
const _render = () => {
const layout: any = {
name: 'dagre'
};
this.cyInstance!.layout(layout).run();
this.props.onRefreshComplete();
};
setTimeout(_render, 100);
}
private renderGraph(graphChanged: boolean) {
const { graph } = this.props;
if (graph === undefined)
return;
this.props.onRefresh();
const _render = () => {
if (graphChanged)
this.expandSet = lodash.cloneDeep(graph.defaultExpandSet);
const graphEl = this.graphElements();
const [remove, add] = this.graphElDifference(this.graphEl, graphEl);
const layout: any = {
name: 'dagre'
};
if (graphEl.length > 100) {
if (remove.size > 0) {
const removedEles = this.cyInstance!.elements().filter(ele => remove.has(ele.id()));
this.cyInstance!.remove(removedEles);
}
if (add.length > 0) {
const eles = this.cyInstance!.add(add);
this.cyInstance!.json({
elements: graphEl
});
layout.fit = false;
eles.layout(layout).run();
}
} else {
this.cyInstance!.json({
elements: graphEl
});
this.cyInstance!.layout(layout).run();
}
this.applyMutableWeight();
this.graphEl = graphEl;
this.props.onRefreshComplete();
};
if (graph.nodes.length > 100)
setTimeout(_render, 100);
else
_render();
}
render() {
return (
<div className='container' ref={this.container}
style={{
left: 0,
top: 0,
position: 'absolute',
width: this.props.width - 15,
height: this.props.height,
overflow: 'hidden'
}}>
</div>
);
}
}
src/nasui/src/graphUtils.ts 0 → 100644
View file @ 75028bd7
function path2module(path: any[]): string {
return path.map(p => p.name ? `${p.type}[${p.name}]` : p.type).join('/');
}
function opName(rawName: string): string {
if (rawName.includes('::')) {
return rawName.split('::')[1];
} else {
return rawName;
}
}
export class NodeTs {
readonly id: string;
readonly tail: string;
parent: NodeTs | undefined;
children: NodeTs[];
op: string;
attributes: string;
constructor(id: string, tail: string, op: string, attributes: string) {
this.children = [];
this.id = id;
this.tail = tail;
this.parent = undefined;
this.op = op;
this.attributes = attributes;
}
descendants(leafOnly: boolean): NodeTs[] {
// return all descendants includinng itself
const result: NodeTs[] = [];
if (!leafOnly || this.isChildless())
result.push(this);
for (const child of this.children) {
const childDesc = child.descendants(leafOnly);
if (childDesc.length > 0)
result.push(...childDesc);
}
return result;
}
isChildless(): boolean {
return this.children.length === 0;
}
isParent(): boolean {
return this.children.length > 0;
}
};
export class Edge {
readonly source: NodeTs;
readonly target: NodeTs;
readonly id: string;
constructor(source: NodeTs, target: NodeTs) {
this.source = source;
this.target = target;
this.id = JSON.stringify([this.source.id, this.target.id]);
}
};
interface NodeSummary {
name: string,
nodeCount: number,
edgeCount: number,
inputs: string[],
outputs: string[],
attributes: string,
op: string
};
export class Graph {
root: NodeTs;
nodes: NodeTs[];
edges: Edge[];
defaultExpandSet: Set<string>;
private id2idx: Map<string, number>;
private edgeId2idx: Map<string, number>;
private forwardGraph: Map<string, string[]>;
private backwardGraph: Map<string, string[]>;
private node2edge: Map<string, Edge[]>;
private mutableEdges: Map<string, Edge[][]>;
private build() {
this.id2idx.clear();
this.nodes.forEach((node, i) => {
this.id2idx.set(node.id, i);
});
this.edgeId2idx.clear();
this.edges.forEach((edge, i) => {
this.edgeId2idx.set(edge.id, i);
});
this.forwardGraph.clear();
this.backwardGraph.clear();
this.node2edge.clear();
this.edges.forEach(edge => {
if (!this.forwardGraph.has(edge.source.id))
this.forwardGraph.set(edge.source.id, []);
this.forwardGraph.get(edge.source.id)!.push(edge.target.id);
if (!this.backwardGraph.has(edge.target.id))
this.backwardGraph.set(edge.target.id, []);
this.backwardGraph.get(edge.target.id)!.push(edge.source.id);
if (!this.node2edge.has(edge.source.id))
this.node2edge.set(edge.source.id, []);
if (!this.node2edge.has(edge.target.id))
this.node2edge.set(edge.target.id, []);
this.node2edge.get(edge.source.id)!.push(edge);
this.node2edge.get(edge.target.id)!.push(edge);
});
this.root.children = this.nodes.filter(node => node.parent === undefined);
// won't set parent for these nodes, leave them as undefined
this.nodes.forEach(node => {
node.children = node.children.filter(child => this.getNodeById(child.id) !== undefined);
});
}
getNodeById(id: string): NodeTs | undefined {
const idx = this.id2idx.get(id);
if (idx === undefined) return undefined;
return this.nodes[idx];
}
getEdgeById(source: string, target: string): Edge | undefined {
const idx = this.edgeId2idx.get(JSON.stringify([source, target]));
if (idx === undefined) return undefined;
return this.edges[idx];
}
constructor(graphData: any, eliminateSidechains: boolean) {
this.id2idx = new Map<string, number>();
this.edgeId2idx = new Map<string, number>();
this.forwardGraph = new Map<string, string[]>();
this.backwardGraph = new Map<string, string[]>();
this.node2edge = new Map<string, Edge[]>();
this.root = new NodeTs('', '', '', '');
const cluster = new Map<string, NodeTs>();
const parentMap = new Map<string, string>();
this.nodes = graphData.node.map((node: any): NodeTs => {
const split = node.name.split('/');
const attr = node.hasOwnProperty('attr') ? atob(node.attr.attr.s) : '';
if (split.length === 1) {
return new NodeTs(node.name, node.name, opName(node.op), attr);
} else {
parentMap.set(node.name, split.slice(0, -1).join('/'));
// create clusters
for (let i = 1; i < split.length; ++i) {
const name = split.slice(0, i).join('/');
if (!cluster.has(name)) {
const parent = i > 1 ? split.slice(0, i - 1).join('/') : '';
const tail = split[i - 1];
cluster.set(name, new NodeTs(name, tail, '', ''));
parentMap.set(name, parent);
}
}
return new NodeTs(node.name, split.slice(-1)[0], opName(node.op), attr);
}
});
cluster.forEach(node => this.nodes.push(node));
this.nodes.forEach((node, i) => {
this.id2idx.set(node.id, i);
});
parentMap.forEach((parent, child) => {
const [childNode, parentNode] = [child, parent].map(this.getNodeById.bind(this));
if (childNode !== undefined && parentNode !== undefined) {
childNode.parent = parentNode;
parentNode.children.push(childNode);
}
});
// build edges
this.edges = [];
graphData.node.forEach((node: any) => {
if (!node.hasOwnProperty('input')) return;
const target = this.getNodeById(node.name);
if (target === undefined) return;
node.input.forEach((input: string) => {
const source = this.getNodeById(input);
if (source !== undefined) {
this.edges.push(new Edge(source, target));
}
})
})
this.build();
if (eliminateSidechains) {
this.eliminateSidechains();
}
this.defaultExpandSet = this.getDefaultExpandSet(graphData.mutable);
this.mutableEdges = this.inferMutableEdges(graphData.mutable);
}
private eliminateSidechains(): void {
const sources = this.nodes
.map(node => node.id)
.filter(id => id.startsWith('input'));
const visitedNodes = new Set(sources);
const dfsStack = sources;
while (dfsStack.length > 0) {
const u = dfsStack.pop()!;
if (this.forwardGraph.has(u)) {
this.forwardGraph.get(u)!.forEach((v: string) => {
if (!visitedNodes.has(v)) {
visitedNodes.add(v);
dfsStack.push(v);
}
});
}
}
const compoundCheck = (node: NodeTs) => {
if (node.isChildless())
return visitedNodes.has(node.id);
for (const child of node.children)
if (compoundCheck(child))
visitedNodes.add(node.id);
return visitedNodes.has(node.id);
}
compoundCheck(this.root);
this.nodes = this.nodes.filter(node => visitedNodes.has(node.id));
this.edges = this.edges.filter(edge =>
visitedNodes.has(edge.source.id) && visitedNodes.has(edge.target.id));
this.build();
}
private getDefaultExpandSet(graphDataMutable: any): Set<string> {
// if multiple, only expand first
const whitelistModuleList = Object.values(graphDataMutable)
.filter(Boolean)
.map((paths: any) => path2module(paths[0]));
const whitelistModule = new Set(whitelistModuleList);
const result = new Set<string>();
const dfs = (node: NodeTs): number => {
// node with mutableCount greater than 0 won't be collapsed
let mutableCount = 0;
if (node.id === '') {
// root node
mutableCount++;
} else if (whitelistModule.has(node.id)) {
mutableCount++;
} else if (node.parent !== undefined && whitelistModule.has(node.parent.id)) {
mutableCount++;
}
mutableCount += node.children.map(child => dfs(child)).reduce((a, b) => a + b, 0);
if (mutableCount > 0 && node.isParent())
result.add(node.id);
return mutableCount;
};
dfs(this.root);
return result;
}
private inferMutableModule(moduleName: string): Edge[][] {
let inputs: string[] | undefined = undefined;
let listConstructNode: string | undefined = undefined;
const moduleNode = this.getNodeById(moduleName);
if (moduleNode === undefined) return [];
for (const node of moduleNode.children)
if (node.op === 'ListConstruct') {
inputs = this.backwardGraph.get(node.id);
listConstructNode = node.id;
break;
}
if (inputs === undefined || listConstructNode === undefined)
return [];
return inputs.map((input: string): Edge[] => {
const visitedNodes = new Set<string>();
const edgeSet: Edge[] = [];
const dfs = (node: string, backward: boolean) => {
const nodeData = this.getNodeById(node)!;
if (visitedNodes.has(node)) return;
visitedNodes.add(node);
if (nodeData.parent === undefined || !nodeData.parent.id.startsWith(moduleName)) {
// in another module now
return;
}
const g = backward ? this.backwardGraph : this.forwardGraph;
const glist = g.get(node);
if (glist !== undefined) {
glist
.forEach((to: string) => {
edgeSet.push(backward ?
this.getEdgeById(to, node)! :
this.getEdgeById(node, to)!);
dfs(to, backward);
});
}
};
edgeSet.push(this.getEdgeById(input, listConstructNode!)!);
dfs(input, true);
visitedNodes.clear();
dfs(listConstructNode!, false);
return edgeSet;
});
}
private inferMutableEdges(graphDataMutable: any): Map<string, Edge[][]> {
const result = new Map<string, Edge[][]>();
Object.entries(graphDataMutable).forEach(obj => {
const [key, paths] = obj;
const modules = (paths as any[]).map(path2module);
const moduleEdge = modules.map(this.inferMutableModule.bind(this));
const edges: Edge[][] = [];
for (let i = 0; ; ++i) {
if (moduleEdge.filter(me => i < me.length).length === 0)
break;
edges.push([]);
moduleEdge
.filter(me => i < me.length)
.forEach(me => edges[i].push(...me[i]));
}
result.set(key, edges);
});
return result;
}
private connectedEdges(node: string): Edge[] {
const result = this.node2edge.get(node);
if (result === undefined)
return [];
return result;
}
nodeSummary(node: NodeTs | string): NodeSummary | undefined {
if (typeof node === 'string') {
const nodeData = this.getNodeById(node);
if (nodeData === undefined) return undefined;
return this.nodeSummary(nodeData);
}
const descendants = node.descendants(false);
const descendantSet = new Set(descendants.map(node => node.id));
const inputs = new Set<string>();
const outputs = new Set<string>();
let domesticEdges = 0;
for (const edge of this.edges) {
const [source, target] = [edge.source.id, edge.target.id];
if (descendantSet.has(target) && !descendantSet.has(source))
inputs.add(source);
if (descendantSet.has(source) && !descendantSet.has(target))
outputs.add(target);
if (descendantSet.has(source) && descendantSet.has(target))
domesticEdges++;
}
return {
name: node.id,
nodeCount: descendants.length,
edgeCount: domesticEdges,
inputs: Array.from(inputs),
outputs: Array.from(outputs),
attributes: node.attributes,
op: node.op
}
}
weightFromMutables(mutable: any): Map<string, number> {
const elemWeight = new Map<string, number>();
Object.entries(mutable).forEach(entry => {
const key = entry[0];
const weights = entry[1] as number[];
this.mutableEdges.get(key)!.forEach((edges: any, i: number) => {
edges.forEach((edge: any) => {
if (elemWeight.has(edge.id)) {
elemWeight.set(edge.id, elemWeight.get(edge.id)! + weights[i]);
} else {
elemWeight.set(edge.id, weights[i]);
}
})
});
});
this.nodes.forEach(node => {
const edges = this.connectedEdges(node.id);
const relatedEdges = edges.filter(edge => elemWeight.has(edge.id));
if (relatedEdges.length > 0) {
if (relatedEdges.length < edges.length) {
elemWeight.set(node.id, 1.);
} else {
// all related edge
const nw = edges.map(edge => elemWeight.get(edge.id)!)
.reduce((a, b) => Math.max(a, b));
elemWeight.set(node.id, nw);
}
}
});
elemWeight.forEach((v, k) => elemWeight.set(k, Math.min(v, 1.)));
// set compound weight
const gatherWeightsFromChildren = (node: NodeTs): number | undefined => {
if (node.isParent()) {
const childrenWeights =
node.children.map(gatherWeightsFromChildren)
.filter(val => val !== undefined);
if (childrenWeights.length > 0) {
const nw = childrenWeights.reduce((a, b) => Math.max(a!, b!));
elemWeight.set(node.id, nw!);
return nw;
} else {
return undefined;
}
} else {
return elemWeight.get(node.id);
}
};
gatherWeightsFromChildren(this.root);
return elemWeight;
}
};
src/nasui/src/index.css 0 → 100644
View file @ 75028bd7
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
code {
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
monospace;
}
src/nasui/src/index.tsx 0 → 100644
View file @ 75028bd7
import React from 'react';
import ReactDOM from 'react-dom';
import './index.css';
import App from './App';
import * as serviceWorker from './serviceWorker';
ReactDOM.render(<App />, document.getElementById('root'));
// If you want your app to work offline and load faster, you can change
// unregister() to register() below. Note this comes with some pitfalls.
// Learn more about service workers: https://bit.ly/CRA-PWA
serviceWorker.unregister();
src/nasui/src/react-app-env.d.ts 0 → 100644
View file @ 75028bd7
/// <reference types="react-scripts" />
src/nasui/src/serviceWorker.ts 0 → 100644
View file @ 75028bd7
// This optional code is used to register a service worker.
// register() is not called by default.
// This lets the app load faster on subsequent visits in production, and gives
// it offline capabilities. However, it also means that developers (and users)
// will only see deployed updates on subsequent visits to a page, after all the
// existing tabs open on the page have been closed, since previously cached
// resources are updated in the background.
// To learn more about the benefits of this model and instructions on how to
// opt-in, read https://bit.ly/CRA-PWA
const isLocalhost = Boolean(
window.location.hostname === 'localhost' ||
// [::1] is the IPv6 localhost address.
window.location.hostname === '[::1]' ||
// 127.0.0.0/8 are considered localhost for IPv4.
window.location.hostname.match(
/^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
)
);
type Config = {
onSuccess?: (registration: ServiceWorkerRegistration) => void;
onUpdate?: (registration: ServiceWorkerRegistration) => void;
};
export function register(config?: Config) {
if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
// The URL constructor is available in all browsers that support SW.
const publicUrl = new URL(
process.env.PUBLIC_URL,
window.location.href
);
if (publicUrl.origin !== window.location.origin) {
// Our service worker won't work if PUBLIC_URL is on a different origin
// from what our page is served on. This might happen if a CDN is used to
// serve assets; see https://github.com/facebook/create-react-app/issues/2374
return;
}
window.addEventListener('load', () => {
const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
if (isLocalhost) {
// This is running on localhost. Let's check if a service worker still exists or not.
checkValidServiceWorker(swUrl, config);
// Add some additional logging to localhost, pointing developers to the
// service worker/PWA documentation.
navigator.serviceWorker.ready.then(() => {
console.log(
'This web app is being served cache-first by a service ' +
'worker. To learn more, visit https://bit.ly/CRA-PWA'
);
});
} else {
// Is not localhost. Just register service worker
registerValidSW(swUrl, config);
}
});
}
}
function registerValidSW(swUrl: string, config?: Config) {
navigator.serviceWorker
.register(swUrl)
.then(registration => {
registration.onupdatefound = () => {
const installingWorker = registration.installing;
if (installingWorker == null) {
return;
}
installingWorker.onstatechange = () => {
if (installingWorker.state === 'installed') {
if (navigator.serviceWorker.controller) {
// At this point, the updated precached content has been fetched,
// but the previous service worker will still serve the older
// content until all client tabs are closed.
console.log(
'New content is available and will be used when all ' +
'tabs for this page are closed. See https://bit.ly/CRA-PWA.'
);
// Execute callback
if (config && config.onUpdate) {
config.onUpdate(registration);
}
} else {
// At this point, everything has been precached.
// It's the perfect time to display a
// "Content is cached for offline use." message.
console.log('Content is cached for offline use.');
// Execute callback
if (config && config.onSuccess) {
config.onSuccess(registration);
}
}
}
};
};
})
.catch(error => {
console.error('Error during service worker registration:', error);
});
}
function checkValidServiceWorker(swUrl: string, config?: Config) {
// Check if the service worker can be found. If it can't reload the page.
fetch(swUrl, {
headers: { 'Service-Worker': 'script' }
})
.then(response => {
// Ensure service worker exists, and that we really are getting a JS file.
const contentType = response.headers.get('content-type');
if (
response.status === 404 ||
(contentType != null && contentType.indexOf('javascript') === -1)
) {
// No service worker found. Probably a different app. Reload the page.
navigator.serviceWorker.ready.then(registration => {
registration.unregister().then(() => {
window.location.reload();
});
});
} else {
// Service worker found. Proceed as normal.
registerValidSW(swUrl, config);
}
})
.catch(() => {
console.log(
'No internet connection found. App is running in offline mode.'
);
});
}
export function unregister() {
if ('serviceWorker' in navigator) {
navigator.serviceWorker.ready
.then(registration => {
registration.unregister();
})
.catch(error => {
console.error(error.message);
});
}
}
src/nasui/tsconfig.json 0 → 100644
View file @ 75028bd7
{
"compilerOptions": {
"target": "es5",
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"typeRoots": [
"./types",
"./node_modules/@types"
],
"allowJs": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react"
},
"include": [
"src"
]
}
src/nasui/types/cytoscape-dagre/index.d.ts 0 → 100644
View file @ 75028bd7
declare module 'cytoscape-dagre';
src/nasui/yarn.lock 0 → 100644
View file @ 75028bd7
This source diff could not be displayed because it is too large. You can view the blob instead.
src/nni_manager/common/observableTimer.ts
View file @ 75028bd7
...@@ -19,7 +19,7 @@ class ObservableTimer { ...@@ -19,7 +19,7 @@ class ObservableTimer {
} }
public unsubscribe( subscription: Rx.IDisposable): void { public unsubscribe( subscription: Rx.IDisposable): void {
if(typeof subscription !== undefined) { if(typeof subscription !== 'undefined') {
subscription.dispose(); subscription.dispose();
} }
} }
......
src/nni_manager/main.ts
View file @ 75028bd7
...@@ -26,6 +26,7 @@ import { PAIYarnTrainingService } from './training_service/pai/paiYarn/paiYarnTr ...@@ -26,6 +26,7 @@ import { PAIYarnTrainingService } from './training_service/pai/paiYarn/paiYarnTr
import { import {
RemoteMachineTrainingService RemoteMachineTrainingService
} from './training_service/remote_machine/remoteMachineTrainingService'; } from './training_service/remote_machine/remoteMachineTrainingService';
import { DLTSTrainingService } from './training_service/dlts/dltsTrainingService';
function initStartupInfo( function initStartupInfo(
startExpMode: string, resumeExperimentId: string, basePort: number, startExpMode: string, resumeExperimentId: string, basePort: number,
...@@ -60,6 +61,10 @@ async function initContainer(foreground: boolean, platformMode: string, logFileN ...@@ -60,6 +61,10 @@ async function initContainer(foreground: boolean, platformMode: string, logFileN
Container.bind(TrainingService) Container.bind(TrainingService)
.to(FrameworkControllerTrainingService) .to(FrameworkControllerTrainingService)
.scope(Scope.Singleton); .scope(Scope.Singleton);
} else if (platformMode === 'dlts') {
Container.bind(TrainingService)
.to(DLTSTrainingService)
.scope(Scope.Singleton);
} else { } else {
throw new Error(`Error: unsupported mode: ${platformMode}`); throw new Error(`Error: unsupported mode: ${platformMode}`);
} }
...@@ -108,7 +113,7 @@ const foreground: boolean = foregroundArg.toLowerCase() === 'true' ? true : fals ...@@ -108,7 +113,7 @@ const foreground: boolean = foregroundArg.toLowerCase() === 'true' ? true : fals
const port: number = parseInt(strPort, 10); const port: number = parseInt(strPort, 10);
const mode: string = parseArg(['--mode', '-m']); const mode: string = parseArg(['--mode', '-m']);
if (!['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn'].includes(mode)) { if (!['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts'].includes(mode)) {
console.log(`FATAL: unknown mode: ${mode}`); console.log(`FATAL: unknown mode: ${mode}`);
usage(); usage();
process.exit(1); process.exit(1);
......
src/nni_manager/rest_server/restValidationSchemas.ts
View file @ 75028bd7
...@@ -140,6 +140,15 @@ export namespace ValidationSchemas { ...@@ -140,6 +140,15 @@ export namespace ValidationSchemas {
}), }),
uploadRetryCount: joi.number().min(1) uploadRetryCount: joi.number().min(1)
}), }),
dlts_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
dashboard: joi.string().min(1),
cluster: joi.string().min(1),
team: joi.string().min(1),
email: joi.string().min(1),
password: joi.string().min(1)
}),
nni_manager_ip: joi.object({ // eslint-disable-line @typescript-eslint/camelcase nni_manager_ip: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
nniManagerIp: joi.string().min(1) nniManagerIp: joi.string().min(1)
}) })
......
src/nni_manager/training_service/common/trialConfigMetadataKey.ts
View file @ 75028bd7
...@@ -18,6 +18,7 @@ export enum TrialConfigMetadataKey { ...@@ -18,6 +18,7 @@ export enum TrialConfigMetadataKey {
KUBEFLOW_CLUSTER_CONFIG = 'kubeflow_config', KUBEFLOW_CLUSTER_CONFIG = 'kubeflow_config',
NNI_MANAGER_IP = 'nni_manager_ip', NNI_MANAGER_IP = 'nni_manager_ip',
FRAMEWORKCONTROLLER_CLUSTER_CONFIG = 'frameworkcontroller_config', FRAMEWORKCONTROLLER_CLUSTER_CONFIG = 'frameworkcontroller_config',
DLTS_CLUSTER_CONFIG = 'dlts_config',
VERSION_CHECK = 'version_check', VERSION_CHECK = 'version_check',
LOG_COLLECTION = 'log_collection' LOG_COLLECTION = 'log_collection'
} }
src/nni_manager/training_service/dlts/dltsClusterConfig.ts 0 → 100644
View file @ 75028bd7
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export interface DLTSClusterConfig {
dashboard: string;
cluster: string;
team: string;
email: string;
password: string;
gpuType?: string;
}
src/nni_manager/training_service/dlts/dltsData.ts 0 → 100644
View file @ 75028bd7
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export const DLTS_TRIAL_COMMAND_FORMAT: string =
`export NNI_PLATFORM=dlts NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \
&& cd $NNI_SYS_DIR && sh install_nni.sh \
&& cd '{6}' && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' \
--nnimanager_ip '{8}' --nnimanager_port '{9}' --nni_manager_version '{10}' --log_collection '{11}'`;
src/nni_manager/training_service/dlts/dltsJobConfig.ts 0 → 100644
View file @ 75028bd7
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import { DLTSClusterConfig } from "./dltsClusterConfig";
export class DLTSJobConfig {
public readonly team: string;
public readonly userName: string;
public readonly vcName: string;
public readonly gpuType: string;
public readonly jobType = "training";
public readonly jobtrainingtype = "RegularJob";
public readonly ssh = false;
public readonly ipython = false;
public readonly tensorboard = false;
public readonly workPath = '';
public readonly enableworkpath = true;
public readonly dataPath = '';
public readonly enabledatapath = false;
public readonly jobPath = '';
public readonly enablejobpath = true;
public readonly mountpoints = [];
public readonly env = [{ name: 'TMPDIR', value: '$HOME/tmp' }]
public readonly hostNetwork = false;
public readonly useGPUTopology = false;
public readonly isPrivileged = false;
public readonly hostIPC = false;
public readonly preemptionAllowed = "False"
public constructor(
clusterConfig: DLTSClusterConfig,
public readonly jobName: string,
public readonly resourcegpu: number,
public readonly image: string,
public readonly cmd: string,
public readonly interactivePorts: number[],
) {
if (clusterConfig.gpuType === undefined) {
throw Error('GPU type not fetched')
}
this.vcName = this.team = clusterConfig.team
this.gpuType = clusterConfig.gpuType
this.userName = clusterConfig.email
}
}
src/nni_manager/training_service/dlts/dltsJobRestServer.ts 0 → 100644
View file @ 75028bd7
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Request, Response, Router } from 'express';
import { Inject } from 'typescript-ioc';
import * as component from '../../common/component';
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { DLTSTrainingService } from './dltsTrainingService';
export interface ParameterFileMeta {
readonly experimentId: string;
readonly trialId: string;
readonly filePath: string;
}
/**
* DLTS Training service Rest server, provides rest API to support DLTS job metrics update
*
*/
@component.Singleton
export class DLTSJobRestServer extends ClusterJobRestServer {
private parameterFileMetaList: ParameterFileMeta[] = [];
@Inject
private readonly dltsTrainingService: DLTSTrainingService;
/**
* constructor to provide NNIRestServer's own rest property, e.g. port
*/
constructor() {
super();
this.dltsTrainingService = component.get(DLTSTrainingService);
}
// tslint:disable-next-line:no-any
protected handleTrialMetrics(jobId: string, metrics: any[]): void {
// Split metrics array into single metric, then emit
// Warning: If not split metrics into single ones, the behavior will be UNKNOWN
for (const singleMetric of metrics) {
this.dltsTrainingService.MetricsEmitter.emit('metric', {
id : jobId,
data : singleMetric
});
}
}
protected createRestHandler(): Router {
const router: Router = super.createRestHandler();
router.post(`/parameter-file-meta`, (req: Request, res: Response) => {
try {
this.log.info(`POST /parameter-file-meta, body is ${JSON.stringify(req.body)}`);
this.parameterFileMetaList.push(req.body);
res.send();
} catch (err) {
this.log.error(`POST parameter-file-meta error: ${err}`);
res.status(500);
res.send(err.message);
}
});
router.get(`/parameter-file-meta`, (req: Request, res: Response) => {
try {
this.log.info(`GET /parameter-file-meta`);
res.send(this.parameterFileMetaList);
} catch (err) {
this.log.error(`GET parameter-file-meta error: ${err}`);
res.status(500);
res.send(err.message);
}
});
return router;
}
}
src/nni_manager/training_service/dlts/dltsTrainingService.ts 0 → 100644
View file @ 75028bd7
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as path from 'path';
import * as request from 'request';
import * as component from '../../common/component';
import { EventEmitter } from 'events';
import { String } from 'typescript-string-operations';
import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log';
import {
NNIManagerIpConfig, TrainingService,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService';
import { DLTS_TRIAL_COMMAND_FORMAT } from './dltsData';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { execMkdir, validateCodeDir } from '../common/util';
import { delay, uniqueString, getIPV4Address, getExperimentRootDir, getVersion, generateParamFileName } from '../../common/utils';
import { DLTSJobRestServer } from './dltsJobRestServer';
import { TrialConfigMetadataKey } from '../../training_service/common/trialConfigMetadataKey';
import { DLTSJobConfig } from './dltsJobConfig';
import { DLTSClusterConfig } from './dltsClusterConfig';
import { DLTSTrialConfig } from './dltsTrialConfig';
import { DLTSTrialJobDetail } from './dltsTrialJobDetail';
@component.Singleton
class DLTSTrainingService implements TrainingService {
private readonly log!: Logger;
private readonly metricsEmitter: EventEmitter;
//private readonly expRootDir: string;
private readonly jobQueue: string[];
private stopping: boolean = false;
private readonly experimentId!: string;
private versionCheck: boolean = true;
private logCollection: string = 'none';
private isMultiPhase: boolean = false;
private dltsRestServerPort?: number;
private readonly trialJobsMap: Map<string, DLTSTrialJobDetail>;
private nniManagerIpConfig?: NNIManagerIpConfig;
private dltsClusterConfig?: DLTSClusterConfig;
private dltsTrialConfig?: DLTSTrialConfig;
constructor() {
this.log = getLogger();
this.metricsEmitter = new EventEmitter();
this.trialJobsMap = new Map();
this.jobQueue = [];
this.experimentId = getExperimentId();
this.log.info('Construct DLTS training service.');
}
public async run(): Promise<void> {
this.log.info('Run DLTS training service.');
const restServer: DLTSJobRestServer = component.get(DLTSJobRestServer);
await restServer.start();
restServer.setEnableVersionCheck = this.versionCheck;
this.log.info(`DLTS Training service rest server listening on: ${restServer.endPoint}`);
await Promise.all([
this.statusCheckingLoop(),
this.submitJobLoop()]);
this.log.info('DLTS training service exit.');
}
private async statusCheckingLoop(): Promise<void> {
while (!this.stopping) {
const updateDLTSTrialJobs: Promise<void>[] = [];
for (const [trialJobId, dltsTrialJob] of this.trialJobsMap) {
updateDLTSTrialJobs.push(this.getDLTSTrialJobInfo(dltsTrialJob));
}
await Promise.all(updateDLTSTrialJobs);
// Calcel paused dlts job
const cancelPausedJobPromises: Promise<void>[] = [];
for (const [trialJobId, dltsTrialJob] of this.trialJobsMap) {
if (dltsTrialJob.dltsPaused && dltsTrialJob.status === 'RUNNING') {
cancelPausedJobPromises.push(this.cancelTrialJob(trialJobId));
}
}
await Promise.all(cancelPausedJobPromises);
const restServer: DLTSJobRestServer = component.get(DLTSJobRestServer);
if (restServer.getErrorMessage !== undefined) {
throw new Error(restServer.getErrorMessage);
}
await delay(3000);
}
}
private async getDLTSTrialJobInfo(dltsTrialJob: DLTSTrialJobDetail): Promise<void> {
if (this.dltsClusterConfig == null) {
throw Error('Cluster config is not set');
}
const requestOptions: request.Options = {
uri: `${this.dltsClusterConfig.dashboard}api/v2/clusters/${this.dltsClusterConfig.cluster}/jobs/${dltsTrialJob.dltsJobId}`,
qs: {
email: this.dltsClusterConfig.email,
password: this.dltsClusterConfig.password
},
json: true
};
const body = await new Promise((resolve, reject) => {
request(requestOptions, (error, response, body) => {
if (error != null) {
reject(error)
} else {
resolve(body)
}
})
}) as any;
void ((): void => {
switch (body['jobStatus']) {
case 'unapproved':
case 'queued':
case 'scheduling':
dltsTrialJob.status = "WAITING";
break;
case 'running':
dltsTrialJob.status = "RUNNING";
if (dltsTrialJob.startTime === undefined) {
dltsTrialJob.startTime = Date.parse(body['jobStatusDetail'][0]['startedAt'])
}
if (dltsTrialJob.url === undefined) {
dltsTrialJob.url = `${this.dltsClusterConfig.dashboard}job/${this.dltsClusterConfig.team}/${this.dltsClusterConfig.cluster}/${dltsTrialJob.dltsJobId}`
}
break;
case 'finished':
dltsTrialJob.status = "SUCCEEDED";
break;
case 'failed':
dltsTrialJob.status = "FAILED";
break;
case 'pausing':
case 'paused':
dltsTrialJob.status = "RUNNING";
dltsTrialJob.dltsPaused = true;
break;
case 'killing':
case 'killed':
if (dltsTrialJob.isEarlyStopped !== undefined) {
dltsTrialJob.status = dltsTrialJob.isEarlyStopped === true
? 'EARLY_STOPPED' : 'USER_CANCELED';
} else {
dltsTrialJob.status = 'SYS_CANCELED';
}
break;
default:
dltsTrialJob.status = "UNKNOWN";
}
}) ();
}
private async submitJobLoop(): Promise<void> {
while (!this.stopping) {
while (!this.stopping && this.jobQueue.length > 0) {
const trialJobId: string = this.jobQueue[0];
this.log.info(`Got job ${trialJobId}`);
if (await this.submitTrialJobToDLTS(trialJobId)) {
// Remove trial job with trialJobId from job queue
this.jobQueue.shift();
} else {
// Break the while loop since failed to submitJob
break;
}
}
await delay(3000);
}
}
public async listTrialJobs(): Promise<TrialJobDetail[]> {
return Array.from(this.trialJobsMap.values());
}
public async getTrialJob(trialJobId: string): Promise<TrialJobDetail> {
const trialJob = this.trialJobsMap.get(trialJobId);
if (trialJob === undefined) {
throw Error(`Trial job ${trialJobId} not found.`)
}
return trialJob
}
public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
this.metricsEmitter.on('metric', listener);
}
public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
this.metricsEmitter.off('metric', listener);
}
public get MetricsEmitter(): EventEmitter {
return this.metricsEmitter;
}
public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const trialJobId: string = uniqueString(5);
const trialWorkingFolder: string = path.join(
'/nni/experiments', getExperimentId(),
'/trials/', trialJobId);
const trialJobDetail = new DLTSTrialJobDetail(
trialJobId, // id
'WAITING', // status
Date.now(), // submitTime
trialWorkingFolder, // workingDirectory
form,
`nni_exp_${this.experimentId}_trial_${trialJobId}`
);
this.trialJobsMap.set(trialJobId, trialJobDetail);
this.jobQueue.push(trialJobId);
return trialJobDetail;
}
public async cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
const trialJobDetail: DLTSTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
if (trialJobDetail === undefined) {
throw Error(`cancelTrialJob: trial job id ${trialJobId} not found`);
}
if (this.dltsClusterConfig === undefined) {
throw Error('DLTS Cluster config is not initialized');
}
const options: request.Options = {
method: 'PUT',
uri: `${this.dltsClusterConfig.dashboard}api/clusters/${this.dltsClusterConfig.cluster}/jobs/${trialJobDetail.dltsJobId}/status`,
qs: {
email: this.dltsClusterConfig.email,
password: this.dltsClusterConfig.password
},
body: {
status: 'killing'
},
json: true
};
// Set trialjobDetail's early stopped field, to mark the job's cancellation source
trialJobDetail.isEarlyStopped = isEarlyStopped;
await new Promise((resolve, reject) => {
request(options, (error: Error, response: request.Response, body: any) => {
if (error) {
reject(error);
} else {
resolve(body);
}
});
});
}
private async getGpuType(): Promise<string> {
if (this.dltsClusterConfig === undefined) {
throw new Error('DLTS Cluster config is not initialized');
}
const gpuRequestOptions: request.Options = {
method: 'GET',
qs: {
email: this.dltsClusterConfig.email,
password: this.dltsClusterConfig.password
},
uri: `${this.dltsClusterConfig.dashboard}api/teams/${this.dltsClusterConfig.team}/clusters/${this.dltsClusterConfig.cluster}`,
json: true
};
return new Promise<string>((resolve, reject) => {
request(gpuRequestOptions, (error, response, data) => {
if (error) {
return reject(error)
}
try {
const metadata = JSON.parse(data['metadata'])
resolve(Object.keys(metadata)[0])
} catch (error) {
reject(error)
}
})
});
}
public async setClusterMetadata(key: string, value: string): Promise<void> {
switch (key) {
case TrialConfigMetadataKey.NNI_MANAGER_IP:
this.nniManagerIpConfig = <NNIManagerIpConfig>JSON.parse(value);
break;
case TrialConfigMetadataKey.DLTS_CLUSTER_CONFIG:
this.dltsClusterConfig = <DLTSClusterConfig>JSON.parse(value);
if (!this.dltsClusterConfig.cluster) {
this.dltsClusterConfig.cluster = '.default'
}
if (!this.dltsClusterConfig.email) {
if (process.env['DLWS_USER_EMAIL']) {
this.dltsClusterConfig.email = process.env['DLWS_USER_EMAIL'] as string
} else {
throw Error('`email` field in `dltsConfig` is not configured.')
}
}
if (!this.dltsClusterConfig.password) {
if (process.env['DLTS_JOB_TOKEN']) {
this.dltsClusterConfig.password = process.env['DLTS_JOB_TOKEN'] as string
} else {
throw Error('`password` field in `dltsConfig` is not configured.')
}
}
if (!this.dltsClusterConfig.team) {
if (process.env['DLWS_VC_NAME']) {
this.dltsClusterConfig.team = process.env['DLWS_VC_NAME'] as string
} else {
throw Error('`team` field in `dltsConfig` is not configured.')
}
}
this.dltsClusterConfig.gpuType = await this.getGpuType();
break;
case TrialConfigMetadataKey.TRIAL_CONFIG:
this.dltsTrialConfig = <DLTSTrialConfig>JSON.parse(value);
// Validate to make sure codeDir doesn't have too many files
try {
await validateCodeDir(this.dltsTrialConfig.codeDir);
} catch (error) {
this.log.error(error);
throw error;
}
break;
case TrialConfigMetadataKey.VERSION_CHECK:
this.versionCheck = (value === 'true' || value === 'True');
break;
case TrialConfigMetadataKey.LOG_COLLECTION:
this.logCollection = value;
break;
case TrialConfigMetadataKey.MULTI_PHASE:
this.isMultiPhase = (value === 'true' || value === 'True');
break;
default:
//Reject for unknown keys
throw new Error(`Uknown key: ${key}`);
}
}
public async getClusterMetadata(key: string): Promise<string> {
return '';
}
public async cleanUp(): Promise<void> {
this.log.info('Stopping DLTS training service...');
this.stopping = true;
const restServer: DLTSJobRestServer = component.get(DLTSJobRestServer);
try {
await restServer.stop();
this.log.info('DLTS Training service rest server stopped successfully.');
return;
} catch (error) {
// tslint:disable-next-line: no-unsafe-any
this.log.error(`DLTS Training service rest server stopped failed, error: ${error.message}`);
throw error;
}
}
private async submitTrialJobToDLTS(trialJobId: string): Promise<boolean> {
const trialJobDetail: DLTSTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
if (trialJobDetail === undefined) {
throw new Error(`Failed to find DLTSTrialJobDetail for job ${trialJobId}`);
}
if (this.dltsClusterConfig === undefined) {
throw new Error('DLTS Cluster config is not initialized');
}
if (this.dltsTrialConfig === undefined) {
throw new Error('trial config is not initialized');
}
if (this.dltsRestServerPort === undefined) {
const restServer: DLTSJobRestServer = component.get(DLTSJobRestServer);
this.dltsRestServerPort = restServer.clusterRestServerPort;
}
// Step 1. Prepare DLTS job configuration
const trialLocalFolder = path.join(getExperimentRootDir(), 'trials-local', trialJobId);
//create tmp trial working folder locally.
await execMkdir(trialLocalFolder);
const runScriptContent: string = CONTAINER_INSTALL_NNI_SHELL_FORMAT;
// Write NNI installation file to local tmp files
await fs.promises.writeFile(path.join(trialLocalFolder, 'install_nni.sh'), runScriptContent, { encoding: 'utf8' });
// Write file content ( parameter.cfg ) to local tmp folders
if (trialJobDetail.form !== undefined) {
await fs.promises.writeFile(
path.join(trialLocalFolder, generateParamFileName(trialJobDetail.form.hyperParameters)),
trialJobDetail.form.hyperParameters.value, { encoding: 'utf8' }
);
}
// tslint:disable-next-line: strict-boolean-expressions
const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address();
const version: string = this.versionCheck ? await getVersion() : '';
const nniDLTSTrialCommand: string = String.Format(
DLTS_TRIAL_COMMAND_FORMAT,
trialLocalFolder,
path.join(trialLocalFolder, 'nnioutput'),
trialJobId,
this.experimentId,
trialJobDetail.form.sequenceId,
false,
this.dltsTrialConfig.codeDir,
this.dltsTrialConfig.command,
nniManagerIp,
this.dltsRestServerPort,
version,
this.logCollection
)
.replace(/\r\n|\n|\r/gm, '');
// Step 2. Submit DLTS job via Rest call
const dltsJobConfig: DLTSJobConfig = new DLTSJobConfig(
this.dltsClusterConfig,
trialJobDetail.dltsJobName,
this.dltsTrialConfig.gpuNum,
this.dltsTrialConfig.image,
nniDLTSTrialCommand,
[]
);
const submitJobRequest: request.Options = {
method: 'POST',
uri: `${this.dltsClusterConfig.dashboard}api/clusters/${this.dltsClusterConfig.cluster}/jobs`,
qs: {
email: this.dltsClusterConfig.email,
password: this.dltsClusterConfig.password
},
body: dltsJobConfig,
json: true
}
const responseData = await new Promise<any>((resolve, reject) => {
request(submitJobRequest, function (error, response, data) {
if (error) {
return reject(error)
} else {
return resolve(data)
}
})
});
trialJobDetail.dltsJobId = responseData['jobId']
return true;
}
public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId);
if (trialJobDetail === undefined) {
throw new Error(`updateTrialJob failed: ${trialJobId} not found`);
}
if (this.dltsClusterConfig === undefined) {
throw new Error('DLTS Cluster config is not initialized');
}
if (this.dltsTrialConfig === undefined) {
throw new Error('DLTS trial config is not initialized');
}
const hyperParameters = form.hyperParameters;
const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId);
const hpFileName: string = generateParamFileName(hyperParameters);
const localFilepath: string = path.join(trialLocalTempFolder, hpFileName);
await fs.promises.writeFile(localFilepath, hyperParameters.value, { encoding: 'utf8' });
const parameterFileMeta = {
experimentId: this.experimentId,
trialId: trialJobId
};
const restServer: DLTSJobRestServer = component.get(DLTSJobRestServer);
const req: request.Options = {
uri: `${restServer.endPoint}${restServer.apiRootUrl}/parameter-file-meta`,
method: 'POST',
json: true,
body: parameterFileMeta
};
await new Promise((resolve, reject) => {
request(req, (err: Error, res: request.Response) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
return trialJobDetail;
}
public get isMultiPhaseJobSupported(): boolean {
return false;
}
}
export { DLTSTrainingService };
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment